Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore: ts-prune

Gergő Móricz8beb05c72671
diff --git a/apps/api/package.json b/apps/api/package.json
index 5f9b70dd2..248727d08 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -59,6 +59,7 @@
"supertest": "^6.3.3",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
+ "ts-prune": "^0.10.3",
"tsc-watch": "^7.1.1",
"tsx": "^4.20.3",
"typescript": "^5.8.3"
@@ -79,6 +80,7 @@
"@devil7softwares/pos": "^1.0.2",
"@dqbd/tiktoken": "^1.0.17",
"@google-cloud/storage": "^7.16.0",
+ "@mendable/firecrawl-rs": "workspace:*",
"@nangohq/node": "^0.40.8",
"@openrouter/ai-sdk-provider": "^0.4.5",
"@opentelemetry/auto-instrumentations-node": "^0.62.0",
@@ -166,8 +168,7 @@
"ws": "^8.18.0",
"x402-express": "^0.5.3",
"xml2js": "^0.6.2",
- "zod": "^3.24.2",
- "@mendable/firecrawl-rs": "workspace:*"
+ "zod": "^3.24.2"
},
"nodemonConfig": {
"ignore": [
@@ -191,7 +192,8 @@
}
},
"lint-staged": {
- "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write"
+ "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write",
+ "*.{js,tsx,ts,tsx}": "ts-prune --skip \"native/\" --error"
},
"packageManager": "[email protected]"
}
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 4642bd0ce..bd7b2e423 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -391,6 +391,9 @@ importers:
ts-node:
specifier: ^10.9.1
version: 10.9.2(@types/[email protected])([email protected])
+ ts-prune:
+ specifier: ^0.10.3
+ version: 0.10.3
tsc-watch:
specifier: ^7.1.1
version: 7.1.1([email protected])
@@ -1968,6 +1971,18 @@ packages:
resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}
engines: {node: ^14.21.3 || >=16}
+ '@nodelib/[email protected]':
+ resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
+ engines: {node: '>= 8'}
+
+ '@nodelib/[email protected]':
+ resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==}
+ engines: {node: '>= 8'}
+
+ '@nodelib/[email protected]':
+ resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
+ engines: {node: '>= 8'}
+
'@octokit/[email protected]':
resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==}
engines: {node: '>= 20'}
@@ -3347,6 +3362,9 @@ packages:
resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}
engines: {node: '>= 10'}
+ '@ts-morph/[email protected]':
+ resolution: {integrity: sha512-4tUmeLyXJnJWvTFOKtcNJ1yh0a3SsTLi2MUoyj8iUNznFRN1ZquaNe7Oukqrnki2FzZkm0J9adCNLDZxUzvj+w==}
+
'@tsconfig/[email protected]':
resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}
@@ -3473,6 +3491,9 @@ packages:
resolution: {integrity: sha512-kK1eBS/Adeyis+3OlBDMeQQuasIDLUYXsi2T15ccNJ0iyUpQ4xDF7svFu3+bGVrI0CMBUclPciz+lsQR3JX3TQ==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==}
+
resolution: {integrity: sha512-+gbBHbNCVGGYw1S9lAIIvrHW47UYOhMIFUsJcMkMrzy1Jf0vulBN3XQIjPgnoOXveMuHnF3b57fXROnY/Or7eg==}
@@ -4088,6 +4109,9 @@ packages:
resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}
engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}
+ resolution: {integrity: sha512-NiujjUFB4SwScJq2bwbYUtXbZhBSlY6vYzm++3Q6oC+U+injTqfPYFK8wS9COOmb2lueqp0ZRB4nK1VYeHgNyw==}
+
resolution: {integrity: sha512-hSo2/tFV29whjFFtVtdS7kHmtUsjfMO1sgwE/d5bhOE4O7Vkj5G1R9lLIqkIprp/+rrvCq3HGvEaOgry7xRcDA==}
@@ -4141,6 +4165,10 @@ packages:
resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==}
+ resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}
@@ -4181,6 +4209,10 @@ packages:
resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==}
engines: {node: '>= 0.10'}
+ resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==}
+ engines: {node: '>=10'}
+
resolution: {integrity: sha512-nPklKJ5qtmY5MdBKw1NiBAoyx5Sa7p2yPpljZyQ7gyCN1m+eMFs9I6CT37Mxt8zvR5L3VzD3DJBE4WQzX3WF4A==}
@@ -4639,6 +4671,10 @@ packages:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
+ resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==}
+ engines: {node: '>=8.6.0'}
+
resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
@@ -4659,6 +4695,9 @@ packages:
resolution: {integrity: sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw==}
+ resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==}
+
resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==}
@@ -4824,6 +4863,10 @@ packages:
resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}
engines: {node: '>= 4.8.0'}
+ resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==}
engines: {node: '>=16 || 14 >=14.18'}
@@ -4976,6 +5019,10 @@ packages:
resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}
+ resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==}
+ engines: {node: '>=6'}
+
resolution: {integrity: sha512-5tCuY9BV8ujfOpwtAGgsTx9CGUapcFMEEyByLv1B+v2+6DhAcw+Zr0nhQT7uwaZ7DiourxFEscghOR8e1aPLQw==}
@@ -5041,6 +5088,10 @@ packages:
resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}
+ resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
+ engines: {node: '>=0.10.0'}
+
resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
engines: {node: '>=8'}
@@ -5061,6 +5112,10 @@ packages:
resolution: {integrity: sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
+ engines: {node: '>=0.10.0'}
+
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
engines: {node: '>=0.12.0'}
@@ -5582,6 +5637,10 @@ packages:
resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}
+ resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}
+ engines: {node: '>= 8'}
+
resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
engines: {node: '>= 0.6'}
@@ -5658,6 +5717,11 @@ packages:
typescript:
optional: true
+ resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}
+ engines: {node: '>=10'}
+ hasBin: true
+
resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==}
engines: {node: '>=10'}
@@ -5731,6 +5795,7 @@ packages:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
+ deprecated: Use your platform's native DOMException instead
resolution: {integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==}
@@ -5922,6 +5987,10 @@ packages:
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
+ resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
+ engines: {node: '>=6'}
+
resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}
@@ -5948,6 +6017,9 @@ packages:
resolution: {integrity: sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==}
+ resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}
+
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
engines: {node: '>=8'}
@@ -5974,6 +6046,10 @@ packages:
resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}
+ resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==}
+ engines: {node: '>=8'}
+
resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}
@@ -6214,6 +6290,9 @@ packages:
resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==}
+ resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
+
resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}
@@ -6318,6 +6397,10 @@ packages:
resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}
+ engines: {node: '>=4'}
+
resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}
engines: {node: '>=8'}
@@ -6345,6 +6428,10 @@ packages:
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
engines: {node: '>= 4'}
+ resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==}
+ engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
+
resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==}
@@ -6358,6 +6445,9 @@ packages:
resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==}
+ resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
+
resolution: {integrity: sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==}
@@ -6635,7 +6725,7 @@ packages:
resolution: {integrity: sha512-6WTxW1EB6yCxV5VFOIPQruWGHqc3yI7hEmZK6h+pyk69Lk/Ut7rLUY6W/ONF2MjBuGjvmMiIpsrVJ2vjrHlslA==}
engines: {node: '>=6.4.0 <13 || >=14'}
- deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net
+ deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net
resolution: {integrity: sha512-7JpaAoX2NGyoFlI9NBh66BQXGONc+uE+MRS5i2iOBKuS4e+ccgMDjATgZldkah+33DakBxDHiss9kvUcGAO8UQ==}
@@ -6648,6 +6738,7 @@ packages:
resolution: {integrity: sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==}
engines: {node: '>=6.4.0'}
+ deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net
resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}
@@ -6752,6 +6843,10 @@ packages:
resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==}
engines: {node: '>= 14.0.0'}
+ resolution: {integrity: sha512-rqy30BSpxPznbbTcAcci90oZ1YR4DqvKcNXNerG5gQBU2v4jk0cygheiul5J6ExIMrgDVuanv/MkGfqZbKrNNg==}
+ engines: {node: 10.* || >= 12.*}
+
resolution: {integrity: sha512-YiHwDhSvCiItoAgsKtoLFCuakDzDsJ1DLDnSouTaTmdOcOwIkSzbLXduaQ6M5DRVhuZC/NYaaZ/mtHbWMv/S6Q==}
engines: {node: ^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0}
@@ -6776,6 +6871,9 @@ packages:
esbuild:
optional: true
+ resolution: {integrity: sha512-pSOfUMx8Ld/WUreoSzvMFQG5i9uEiWIsBYjpU9+TTASOeUa89j5HykomeqVULm1oqWtBdleI3KEFRLrlA3zGIw==}
+
resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}
hasBin: true
@@ -6790,6 +6888,10 @@ packages:
'@swc/wasm':
optional: true
+ resolution: {integrity: sha512-iS47YTbdIcvN8Nh/1BFyziyUqmjXz7GVzWu02RaZXqb+e/3Qe1B7IQ4860krOeCGUeJmterAlaM2FRH0Ue0hjw==}
+ hasBin: true
+
resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}
engines: {node: '>=12.12.0'}
@@ -7261,6 +7363,10 @@ packages:
resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==}
engines: {node: '>= 14.6'}
@@ -9360,6 +9466,18 @@ snapshots:
'@noble/[email protected]': {}
+ '@nodelib/[email protected]':
+ dependencies:
+ '@nodelib/fs.stat': 2.0.5
+ run-parallel: 1.2.0
+
+ '@nodelib/[email protected]': {}
+
+ '@nodelib/[email protected]':
+ dependencies:
+ '@nodelib/fs.scandir': 2.1.5
+ fastq: 1.19.1
+
'@octokit/[email protected]': {}
'@octokit/[email protected]':
@@ -11477,6 +11595,13 @@ snapshots:
'@tootallnate/[email protected]': {}
+ '@ts-morph/[email protected]':
+ dependencies:
+ fast-glob: 3.3.3
+ minimatch: 3.1.2
+ mkdirp: 1.0.4
+ path-browserify: 1.0.1
+
'@tsconfig/[email protected]': {}
'@tsconfig/[email protected]': {}
@@ -11625,6 +11750,8 @@ snapshots:
dependencies:
'@types/node': 20.14.1
+ '@types/[email protected]': {}
+
'@types/[email protected]': {}
@@ -12768,6 +12895,8 @@ snapshots:
+
dependencies:
'@aws-sdk/client-sagemaker': 3.679.0
@@ -12832,6 +12961,8 @@ snapshots:
+
@@ -12864,6 +12995,14 @@ snapshots:
object-assign: 4.1.1
vary: 1.1.2
+ dependencies:
+ '@types/parse-json': 4.0.2
+ import-fresh: 3.3.1
+ parse-json: 5.2.0
+ path-type: 4.0.0
+ yaml: 1.10.2
+
@@ -13350,6 +13489,14 @@ snapshots:
+ dependencies:
+ '@nodelib/fs.stat': 2.0.5
+ '@nodelib/fs.walk': 1.2.8
+ glob-parent: 5.1.2
+ merge2: 1.4.1
+ micromatch: 4.0.8
+
@@ -13364,6 +13511,10 @@ snapshots:
+ dependencies:
+ reusify: 1.1.0
+
dependencies:
bser: 2.1.1
@@ -13556,6 +13707,10 @@ snapshots:
shelljs: 0.8.5
shelljs.exec: 1.1.8
+ dependencies:
+ is-glob: 4.0.3
+
dependencies:
foreground-child: 3.2.1
@@ -13735,6 +13890,11 @@ snapshots:
+ dependencies:
+ parent-module: 1.0.1
+ resolve-from: 4.0.0
+
dependencies:
acorn: 8.15.0
@@ -13807,6 +13967,8 @@ snapshots:
dependencies:
hasown: 2.0.2
+
@@ -13824,6 +13986,10 @@ snapshots:
has-tostringtag: 1.0.2
safe-regex-test: 1.1.0
+ dependencies:
+ is-extglob: 2.1.1
+
@@ -14587,6 +14753,8 @@ snapshots:
+
@@ -14640,6 +14808,8 @@ snapshots:
optionalDependencies:
typescript: 5.8.3
+
@@ -14920,6 +15090,10 @@ snapshots:
+ dependencies:
+ callsites: 3.1.0
+
@@ -14951,6 +15125,8 @@ snapshots:
+
@@ -14968,6 +15144,8 @@ snapshots:
+
dependencies:
through: 2.3.8
@@ -15204,6 +15382,8 @@ snapshots:
+
@@ -15312,6 +15492,8 @@ snapshots:
dependencies:
resolve-from: 5.0.0
+
@@ -15340,6 +15522,8 @@ snapshots:
+
@@ -15359,6 +15543,10 @@ snapshots:
+ dependencies:
+ queue-microtask: 1.2.3
+
@@ -15783,6 +15971,8 @@ snapshots:
+
dependencies:
bs-logger: 0.2.6
@@ -15801,6 +15991,11 @@ snapshots:
'@jest/types': 29.6.3
babel-jest: 29.7.0(@babel/[email protected])
+ dependencies:
+ '@ts-morph/common': 0.12.3
+ code-block-writer: 11.0.3
+
dependencies:
'@cspotcode/source-map-support': 0.8.1
@@ -15819,6 +16014,15 @@ snapshots:
v8-compile-cache-lib: 3.0.1
yn: 3.1.1
+ dependencies:
+ commander: 6.2.1
+ cosmiconfig: 7.1.0
+ json5: 2.2.3
+ lodash: 4.17.21
+ true-myth: 4.1.1
+ ts-morph: 13.0.3
+
dependencies:
cross-spawn: 7.0.6
@@ -16328,6 +16532,8 @@ snapshots:
+
diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.ts
index 4a8d9906e..0d770234d 100644
--- a/apps/api/src/__tests__/snips/v1/lib.ts
+++ b/apps/api/src/__tests__/snips/v1/lib.ts
@@ -25,14 +25,7 @@ import {
} from "../lib";
// Re-export shared utilities for backwards compatibility
-export {
- TEST_URL,
- scrapeTimeout,
- indexCooldown,
- Identity,
- IdmuxRequest,
- idmux,
-};
+export { scrapeTimeout, indexCooldown, Identity, idmux };
export async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {
return await request(TEST_URL)
@@ -207,7 +200,7 @@ export async function asyncCrawlWaitForFinish(
return x.body;
}
-export async function crawlErrors(
+async function crawlErrors(
id: string,
identity: Identity,
): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {
@@ -473,7 +466,7 @@ export async function tokenUsage(
// Concurrency API
// =========================================
-export async function concurrencyCheck(
+async function concurrencyCheck(
identity: Identity,
): Promise<{ concurrency: number; maxConcurrency: number }> {
const x = await request(TEST_URL)
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index ef4103e9a..6ab7c96f1 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -27,14 +27,7 @@ import {
import { SearchV2Response } from "../../../lib/entities";
// Re-export shared utilities for backwards compatibility
-export {
- TEST_URL,
- scrapeTimeout,
- indexCooldown,
- Identity,
- IdmuxRequest,
- idmux,
-};
+export { scrapeTimeout, indexCooldown, Identity, idmux };
// =========================================
// Scrape API
@@ -208,7 +201,7 @@ export async function asyncCrawlWaitForFinish(
return x.body;
}
-export async function crawlErrors(
+async function crawlErrors(
id: string,
identity: Identity,
): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {
@@ -409,7 +402,7 @@ export async function creditUsage(
// Concurrency API
// =========================================
-export async function concurrencyCheck(
+async function concurrencyCheck(
identity: Identity,
): Promise<{ concurrency: number; maxConcurrency: number }> {
const x = await request(TEST_URL)
diff --git a/apps/api/src/control.ts b/apps/api/src/control.ts
deleted file mode 100644
index ba42542ea..000000000
--- a/apps/api/src/control.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-// ! IN CASE OPENAI goes down, then activate the fallback -> true
-export const is_fallback = false;
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index ad2fd5256..c5c116fd6 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -398,7 +398,7 @@ export async function authenticateUser(
})(req, res, mode);
}
-export async function supaAuthenticateUser(
+async function supaAuthenticateUser(
req,
res,
mode?: RateLimiterMode,
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 446e20f0c..5a17c08c8 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -13,7 +13,7 @@ import { getJobFromGCS } from "../../lib/gcs-jobs";
import { scrapeQueue, NuQJob } from "../../services/worker/nuq";
configDotenv();
-export async function getJobs(
+async function getJobs(
crawlId: string,
ids: string[],
): Promise<PseudoJob<any>[]> {
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 065c54c54..820bef824 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -24,7 +24,7 @@ import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
-export async function searchHelper(
+async function searchHelper(
jobId: string,
req: Request,
team_id: string,
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts
index 947e1aa22..682b95de3 100644
--- a/apps/api/src/controllers/v1/deep-research.ts
+++ b/apps/api/src/controllers/v1/deep-research.ts
@@ -5,7 +5,7 @@ import * as Sentry from "@sentry/node";
import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";
import { z } from "zod";
-export const deepResearchRequestSchema = z
+const deepResearchRequestSchema = z
.object({
query: z.string().describe("The query or topic to search for").optional(),
maxDepth: z
@@ -60,9 +60,9 @@ export const deepResearchRequestSchema = z
query: data.topic || data.query, // Use topic as query if provided
}));
-export type DeepResearchRequest = z.infer<typeof deepResearchRequestSchema>;
+type DeepResearchRequest = z.infer<typeof deepResearchRequestSchema>;
-export type DeepResearchResponse =
+type DeepResearchResponse =
| ErrorResponse
| {
success: boolean;
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index cea0cc3bf..32fad784a 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -19,7 +19,7 @@ type ExtractPseudoJob<T> = {
failedReason?: string;
};
-export async function getExtractJob(
+async function getExtractJob(
id: string,
): Promise<ExtractPseudoJob<ExtractResult> | null> {
const [bullJob, dbJob] = await Promise.all([
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 2e9e39972..0400763a8 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -19,7 +19,7 @@ import { logger as _logger } from "../../lib/logger";
import { fromV1ScrapeOptions } from "../v2/types";
import { createWebhookSender, WebhookEvent } from "../../services/webhook";
-export async function oldExtract(
+async function oldExtract(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>,
extractId: string,
diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts
index ccc2145d7..2e6464e73 100644
--- a/apps/api/src/controllers/v1/generate-llmstxt.ts
+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts
@@ -9,7 +9,7 @@ import { getGenerateLlmsTxtQueue } from "../../services/queue-service";
import * as Sentry from "@sentry/node";
import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";
-export type GenerateLLMsTextResponse =
+type GenerateLLMsTextResponse =
| ErrorResponse
| {
success: boolean;
diff --git a/apps/api/src/controllers/v1/liveness.ts b/apps/api/src/controllers/v1/liveness.ts
deleted file mode 100644
index 8ff1a96f9..000000000
--- a/apps/api/src/controllers/v1/liveness.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import { Request, Response } from "express";
-
-export async function livenessController(req: Request, res: Response) {
- //TODO: add checks if the application is live and healthy like checking the redis connection
- res.status(200).json({ status: "ok" });
-}
diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.ts
index 64d30be1f..63e0157fc 100644
--- a/apps/api/src/controllers/v1/queue-status.ts
+++ b/apps/api/src/controllers/v1/queue-status.ts
@@ -10,7 +10,7 @@ import {
getConcurrencyQueueJobsCount,
} from "../../lib/concurrency-limit";
-export type QueueStatusResponse = {
+type QueueStatusResponse = {
success: boolean;
jobsInQueue: number;
activeJobsInQueue: number;
diff --git a/apps/api/src/controllers/v1/readiness.ts b/apps/api/src/controllers/v1/readiness.ts
deleted file mode 100644
index cdb1f02c3..000000000
--- a/apps/api/src/controllers/v1/readiness.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import { Request, Response } from "express";
-
-export async function readinessController(req: Request, res: Response) {
- // TODO: add checks when the application is ready to serve traffic
- res.status(200).json({ status: "ok" });
-}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 0c3cac160..2a1e5c7c7 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -15,7 +15,7 @@ import { ErrorCodes } from "../../lib/error";
import { integrationSchema } from "../../utils/integration";
import { webhookSchema } from "../../services/webhook/schema";
-export type Format =
+type Format =
| "markdown"
| "html"
| "rawHtml"
@@ -70,7 +70,7 @@ export const url = z.preprocess(
const strictMessage =
"Unrecognized key in body -- please review the v1 API documentation for request body changes";
-export const agentExtractModelValue = "fire-1";
+const agentExtractModelValue = "fire-1";
export const isAgentExtractModelValid = (x: string | undefined) =>
x?.toLowerCase() === agentExtractModelValue;
@@ -80,8 +80,6 @@ export const agentOptionsExtract = z
})
.strict(strictMessage);
-export type AgentOptions = z.infer<typeof agentOptionsExtract>;
-
export const extractOptions = z
.object({
mode: z.enum(["llm"]).default("llm"),
@@ -113,7 +111,7 @@ export const extractOptions = z
"Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required.",
}));
-export const extractOptionsWithAgent = z
+const extractOptionsWithAgent = z
.object({
mode: z.enum(["llm"]).default("llm"),
schema: z
@@ -188,7 +186,7 @@ function calculateTotalWaitTime(
return waitFor + actionWaitTime;
}
-export const actionSchema = z.union([
+const actionSchema = z.union([
z
.object({
type: z.literal("wait"),
@@ -258,7 +256,7 @@ export const actionSchema = z.union([
export type Action = z.infer<typeof actionSchema>;
-export const actionsSchema = z
+const actionsSchema = z
.array(actionSchema)
.refine(actions => actions.length <= MAX_ACTIONS, {
message: `Number of actions cannot exceed ${MAX_ACTIONS}`,
@@ -272,7 +270,7 @@ export const actionsSchema = z
const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];
-export const locationSchema = z
+const locationSchema = z
.object({
country: z
.string()
@@ -536,7 +534,7 @@ export const scrapeOptions = baseScrapeOptions
return extractTransform(obj) as typeof obj;
});
-export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;
+type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;
export type ScrapeOptions = BaseScrapeOptions & {
extract?: z.infer<typeof extractOptionsWithAgent>;
@@ -551,7 +549,7 @@ export type ScrapeOptions = BaseScrapeOptions & {
const ajv = new Ajv();
-export const extractV1Options = z
+const extractV1Options = z
.object({
urls: url
.array()
@@ -629,7 +627,6 @@ export const extractV1Options = z
: x.scrapeOptions,
}));
-export type ExtractV1Options = z.infer<typeof extractV1Options>;
export const extractRequestSchema = extractV1Options;
export type ExtractRequest = z.infer<typeof extractRequestSchema>;
export type ExtractRequestInput = z.input<typeof extractRequestSchema>;
@@ -732,7 +729,7 @@ const crawlerOptions = z
// ignoreSitemap?: boolean;
// };
-export type CrawlerOptions = z.infer<typeof crawlerOptions>;
+type CrawlerOptions = z.infer<typeof crawlerOptions>;
export const crawlRequestSchema = crawlerOptions
.extend({
@@ -935,12 +932,6 @@ export type ScrapeResponse =
scrape_id?: string;
};
-export interface ScrapeResponseRequestTest {
- statusCode: number;
- body: ScrapeResponse;
- error?: string;
-}
-
export interface URLTrace {
url: string;
status: "mapped" | "scraped" | "error";
@@ -975,12 +966,6 @@ export interface ExtractResponse {
tokensUsed?: number;
}
-export interface ExtractResponseRequestTest {
- statusCode: number;
- body: ExtractResponse;
- error?: string;
-}
-
export type CrawlResponse =
| ErrorResponse
| {
@@ -1207,7 +1192,7 @@ export function toNewCrawlerOptions(x: any): CrawlerOptions {
};
}
-export function fromLegacyCrawlerOptions(
+function fromLegacyCrawlerOptions(
x: any,
teamId: string,
): {
@@ -1305,26 +1290,6 @@ export function fromLegacyScrapeOptions(
};
}
-export function fromLegacyCombo(
- pageOptions: PageOptions,
- extractorOptions: ExtractorOptions | undefined,
- timeout: number | undefined,
- crawlerOptions: any,
- teamId: string,
-): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
- const { scrapeOptions, internalOptions: i1 } = fromLegacyScrapeOptions(
- pageOptions,
- extractorOptions,
- timeout,
- teamId,
- );
- const { internalOptions: i2 } = fromLegacyCrawlerOptions(
- crawlerOptions,
- teamId,
- );
- return { scrapeOptions, internalOptions: Object.assign(i1, i2) };
-}
-
export function toLegacyDocument(
document: Document,
internalOptions: InternalOptions,
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index bec3b62bc..e3ef33d9a 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -19,7 +19,7 @@ type ExtractPseudoJob<T> = {
failedReason?: string;
};
-export async function getExtractJob(
+async function getExtractJob(
id: string,
): Promise<ExtractPseudoJob<ExtractResult> | null> {
const [bullJob, dbJob] = await Promise.all([
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 849250c5c..aaf966b1a 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -86,7 +86,7 @@ async function queryIndex(
}
}
-export async function getMapResults({
+async function getMapResults({
url,
search,
limit = MAX_MAP_LIMIT,
diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.ts
index bf03dbc29..e52542dce 100644
--- a/apps/api/src/controllers/v2/queue-status.ts
+++ b/apps/api/src/controllers/v2/queue-status.ts
@@ -11,7 +11,7 @@ import {
getConcurrencyQueueJobsCount,
} from "../../lib/concurrency-limit";
-export type QueueStatusResponse = {
+type QueueStatusResponse = {
success: boolean;
jobsInQueue: number;
activeJobsInQueue: number;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 12fc05b8f..dc67932cf 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -20,19 +20,6 @@ import Ajv from "ajv";
import { integrationSchema } from "../../utils/integration";
import { webhookSchema } from "../../services/webhook/schema";
-export type Format =
- | "markdown"
- | "html"
- | "rawHtml"
- | "links"
- | "images"
- | "screenshot"
- | "screenshot@fullPage"
- | "extract"
- | "json"
- | "summary"
- | "changeTracking";
-
export const url = z.preprocess(
x => {
if (!protocolIncluded(x as string)) {
@@ -98,7 +85,7 @@ function calculateTotalWaitTime(
return waitFor + actionWaitTime;
}
-export const actionSchema = z.union([
+const actionSchema = z.union([
z
.object({
type: z.literal("wait"),
@@ -172,9 +159,7 @@ export const actionSchema = z.union([
}),
]);
-export type Action = z.infer<typeof actionSchema>;
-
-export const actionsSchema = z
+const actionsSchema = z
.array(actionSchema)
.refine(actions => actions.length <= MAX_ACTIONS, {
message: `Number of actions cannot exceed ${MAX_ACTIONS}`,
@@ -186,7 +171,7 @@ export const actionsSchema = z
},
);
-export const jsonFormatWithOptions = z
+const jsonFormatWithOptions = z
.object({
type: z.literal("json"),
schema: z.any().optional(),
@@ -196,7 +181,7 @@ export const jsonFormatWithOptions = z
export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;
-export const changeTrackingFormatWithOptions = z
+const changeTrackingFormatWithOptions = z
.object({
type: z.literal("changeTracking"),
prompt: z.string().optional(),
@@ -206,11 +191,11 @@ export const changeTrackingFormatWithOptions = z
})
.strict();
-export type ChangeTrackingFormatWithOptions = z.output<
+type ChangeTrackingFormatWithOptions = z.output<
typeof changeTrackingFormatWithOptions
>;
-export const screenshotFormatWithOptions = z.object({
+const screenshotFormatWithOptions = z.object({
type: z.literal("screenshot"),
fullPage: z.boolean().default(false),
quality: z.number().min(1).max(100).optional(),
@@ -222,11 +207,9 @@ export const screenshotFormatWithOptions = z.object({
.optional(),
});
-export type ScreenshotFormatWithOptions = z.output<
- typeof screenshotFormatWithOptions
->;
+type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;
-export const attributesFormatWithOptions = z
+const attributesFormatWithOptions = z
.object({
type: z.literal("attributes"),
selectors: z
@@ -244,9 +227,7 @@ export const attributesFormatWithOptions = z
})
.strict();
-export type AttributesFormatWithOptions = z.output<
- typeof attributesFormatWithOptions
->;
+type AttributesFormatWithOptions = z.output<typeof attributesFormatWithOptions>;
export type FormatObject =
| { type: "markdown" }
@@ -260,18 +241,18 @@ export type FormatObject =
| ScreenshotFormatWithOptions
| AttributesFormatWithOptions;
-export const pdfParserWithOptions = z
+const pdfParserWithOptions = z
.object({
type: z.literal("pdf"),
maxPages: z.number().int().positive().finite().max(10000).optional(),
})
.strict();
-export const parsersSchema = z
+const parsersSchema = z
.array(z.union([z.literal("pdf"), pdfParserWithOptions]))
.default(["pdf"]);
-export type Parsers = z.infer<typeof parsersSchema>;
+type Parsers = z.infer<typeof parsersSchema>;
export function shouldParsePDF(parsers?: Parsers): boolean {
if (!parsers) return true;
@@ -307,7 +288,7 @@ function transformIframeSelector(selector: string): string {
const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];
-export const locationSchema = z
+const locationSchema = z
.object({
country: z
.string()
@@ -473,7 +454,7 @@ export type ScrapeOptions = BaseScrapeOptions;
const ajv = new Ajv();
-export const extractOptions = z
+const extractOptions = z
.object({
urls: url
.array()
@@ -543,7 +524,6 @@ export const extractOptions = z
: x.scrapeOptions,
}));
-export type ExtractOptions = z.infer<typeof extractOptions>;
export const extractRequestSchema = extractOptions;
export type ExtractRequest = z.infer<typeof extractRequestSchema>;
export type ExtractRequestInput = z.input<typeof extractRequestSchema>;
@@ -623,7 +603,7 @@ const crawlerOptions = z
// ignoreSitemap?: boolean;
// };
-export type CrawlerOptions = z.infer<typeof crawlerOptions>;
+type CrawlerOptions = z.infer<typeof crawlerOptions>;
export const crawlRequestSchema = crawlerOptions
.extend({
@@ -811,12 +791,6 @@ export type ScrapeResponse =
scrape_id?: string;
};
-export interface ScrapeResponseRequestTest {
- statusCode: number;
- body: ScrapeResponse;
- error?: string;
-}
-
export interface URLTrace {
url: string;
status: "mapped" | "scraped" | "error";
@@ -851,12 +825,6 @@ export interface ExtractResponse {
tokensUsed?: number;
}
-export interface ExtractResponseRequestTest {
- statusCode: number;
- body: ExtractResponse;
- error?: string;
-}
-
export type CrawlResponse =
| ErrorResponse
| {
@@ -964,7 +932,7 @@ export type TeamFlags = {
ipWhitelist?: boolean;
} | null;
-export interface RequestWithMaybeACUC<
+interface RequestWithMaybeACUC<
ReqParams = {},
ReqBody = undefined,
ResBody = undefined,
@@ -972,15 +940,6 @@ export interface RequestWithMaybeACUC<
acuc?: AuthCreditUsageChunk;
}
-export interface RequestWithMaybeAuth<
- ReqParams = {},
- ReqBody = undefined,
- ResBody = undefined,
-> extends RequestWithMaybeACUC<ReqParams, ReqBody, ResBody> {
- auth?: AuthObject;
- account?: Account;
-}
-
export interface RequestWithAuth<
ReqParams = {},
ReqBody = undefined,
@@ -990,11 +949,6 @@ export interface RequestWithAuth<
account?: Account;
}
-export interface ResponseWithSentry<ResBody = undefined>
- extends Response<ResBody> {
- sentry?: string;
-}
-
export function toV0CrawlerOptions(x: CrawlerOptions) {
return {
includes: x.includePaths,
@@ -1035,7 +989,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {
};
}
-export function fromV0CrawlerOptions(
+function fromV0CrawlerOptions(
x: any,
teamId: string,
): {
@@ -1257,39 +1211,11 @@ export function fromV0Combo(
return { scrapeOptions, internalOptions: Object.assign(i1, i2) };
}
-export function toLegacyDocument(
- document: Document,
- internalOptions: InternalOptions,
-): V0Document | { url: string } {
- if (internalOptions.v0CrawlOnlyUrls) {
- return { url: document.metadata.sourceURL! };
- }
-
- return {
- content: document.markdown!,
- markdown: document.markdown!,
- html: document.html,
- rawHtml: document.rawHtml,
- linksOnPage: document.links,
- llm_extraction: document.extract,
- metadata: {
- ...document.metadata,
- error: undefined,
- statusCode: undefined,
- pageError: document.metadata.error,
- pageStatusCode: document.metadata.statusCode,
- screenshot: document.screenshot,
- },
- actions: document.actions,
- warning: document.warning,
- };
-}
-
// Search source type definitions
// These allow fine-grained control over each search source type
// Similar to how scrape formats work with jsonFormatWithOptions, etc.
-export const webSearchSourceOptions = z
+const webSearchSourceOptions = z
.object({
type: z.literal("web"),
tbs: z.string().optional(), // Time-based search (e.g., "qdr:d" for past day)
@@ -1300,40 +1226,31 @@ export const webSearchSourceOptions = z
})
.strict();
-export const imagesSearchSourceOptions = z
+const imagesSearchSourceOptions = z
.object({
type: z.literal("images"),
})
.strict();
-export const newsSearchSourceOptions = z
+const newsSearchSourceOptions = z
.object({
type: z.literal("news"),
})
.strict();
-export type WebSearchSourceOptions = z.infer<typeof webSearchSourceOptions>;
-export type ImagesSearchSourceOptions = z.infer<
- typeof imagesSearchSourceOptions
->;
-export type NewsSearchSourceOptions = z.infer<typeof newsSearchSourceOptions>;
-
// Category source type definitions
-export const githubCategoryOptions = z
+const githubCategoryOptions = z
.object({
type: z.literal("github"),
})
.strict();
-export const researchCategoryOptions = z
+const researchCategoryOptions = z
.object({
type: z.literal("research"),
})
.strict();
-export type GithubCategoryOptions = z.infer<typeof githubCategoryOptions>;
-export type ResearchCategoryOptions = z.infer<typeof researchCategoryOptions>;
-
export const searchRequestSchema = z
.object({
query: z.string(),
@@ -1526,7 +1443,7 @@ export type TokenUsage = {
model?: string;
};
-export const generateLLMsTextRequestSchema = z.object({
+const generateLLMsTextRequestSchema = z.object({
url: url.describe("The URL to generate text from"),
maxUrls: z
.number()
@@ -1545,6 +1462,4 @@ export const generateLLMsTextRequestSchema = z.object({
__experimental_stream: z.boolean().optional(),
});
-export type GenerateLLMsTextRequest = z.infer<
- typeof generateLLMsTextRequestSchema
->;
+type GenerateLLMsTextRequest = z.infer<typeof generateLLMsTextRequestSchema>;
diff --git a/apps/api/src/lib/LLM-extraction/helpers.ts b/apps/api/src/lib/LLM-extraction/helpers.ts
deleted file mode 100644
index 2143a32db..000000000
--- a/apps/api/src/lib/LLM-extraction/helpers.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { encoding_for_model } from "@dqbd/tiktoken";
-import { TiktokenModel } from "@dqbd/tiktoken";
-
-// This function calculates the number of tokens in a text string using GPT-3.5-turbo model
-export function numTokensFromString(message: string, model: string): number {
- const encoder = encoding_for_model(model as TiktokenModel);
-
- // Encode the message into tokens
- let tokens: Uint32Array;
- try {
- tokens = encoder.encode(message);
- } catch (error) {
- message = message.replace("<|endoftext|>", "");
- tokens = encoder.encode(message);
- }
-
- // Free the encoder resources after use
- encoder.free();
-
- // Return the number of tokens
- return tokens.length;
-}
diff --git a/apps/api/src/lib/batch-process.ts b/apps/api/src/lib/batch-process.ts
deleted file mode 100644
index 1e4ac7be2..000000000
--- a/apps/api/src/lib/batch-process.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-export async function batchProcess<T>(
- array: T[],
- batchSize: number,
- asyncFunction: (item: T, index: number) => Promise<void>,
-): Promise<void> {
- const batches: T[][] = [];
- for (let i = 0; i < array.length; i += batchSize) {
- const batch = array.slice(i, i + batchSize);
- batches.push(batch);
- }
-
- for (const batch of batches) {
- await Promise.all(batch.map((item, i) => asyncFunction(item, i)));
- }
-}
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index e56721769..e8bbb122c 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -54,14 +54,11 @@ export async function pushConcurrencyLimitActiveJob(
await redisEvictConnection.zadd(constructKey(team_id), now + timeout, id);
}
-export async function removeConcurrencyLimitActiveJob(
- team_id: string,
- id: string,
-) {
+async function removeConcurrencyLimitActiveJob(team_id: string, id: string) {
await redisEvictConnection.zrem(constructKey(team_id), id);
}
-export type ConcurrencyLimitedJob = {
+type ConcurrencyLimitedJob = {
id: string;
data: any;
priority: number;
@@ -78,22 +75,6 @@ export async function cleanOldConcurrencyLimitedJobs(
);
}
-export async function takeConcurrencyLimitedJob(
- team_id: string,
-): Promise<ConcurrencyLimitedJob | null> {
- await cleanOldConcurrencyLimitedJobs(team_id);
- const res = await redisEvictConnection.zmpop(
- 1,
- constructQueueKey(team_id),
- "MIN",
- );
- if (res === null || res === undefined) {
- return null;
- }
-
- return JSON.parse(res[1][0][0]);
-}
-
export async function pushConcurrencyLimitedJob(
team_id: string,
job: ConcurrencyLimitedJob,
@@ -123,7 +104,7 @@ export async function getConcurrencyQueueJobsCount(
);
}
-export async function cleanOldCrawlConcurrencyLimitEntries(
+async function cleanOldCrawlConcurrencyLimitEntries(
crawl_id: string,
now: number = Date.now(),
) {
@@ -158,7 +139,7 @@ export async function pushCrawlConcurrencyLimitActiveJob(
);
}
-export async function removeCrawlConcurrencyLimitActiveJob(
+async function removeCrawlConcurrencyLimitActiveJob(
crawl_id: string,
id: string,
) {
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 0ffc6a97a..78ba0cda2 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -189,7 +189,7 @@ export async function getDoneJobsOrderedUntil(
);
}
-export async function isCrawlFinished(id: string) {
+async function isCrawlFinished(id: string) {
await redisEvictConnection.expire(
"crawl:" + id + ":kickoff:finish",
24 * 60 * 60,
@@ -211,10 +211,6 @@ export async function isCrawlKickoffFinished(id: string) {
);
}
-export async function isCrawlFinishedLocked(id: string) {
- return await redisEvictConnection.exists("crawl:" + id + ":finish");
-}
-
export async function finishCrawlKickoff(id: string) {
await redisEvictConnection.set(
"crawl:" + id + ":kickoff:finish",
@@ -279,10 +275,6 @@ export async function getCrawlJobs(id: string): Promise<string[]> {
return await redisEvictConnection.smembers("crawl:" + id + ":jobs");
}
-export async function getCrawlJobCount(id: string): Promise<number> {
- return await redisEvictConnection.scard("crawl:" + id + ":jobs");
-}
-
export async function getCrawlQualifiedJobCount(id: string): Promise<number> {
return await redisEvictConnection.scard("crawl:" + id + ":jobs_qualified");
}
diff --git a/apps/api/src/lib/deep-research/deep-research-redis.ts b/apps/api/src/lib/deep-research/deep-research-redis.ts
index d53bf5e9b..0135a0f04 100644
--- a/apps/api/src/lib/deep-research/deep-research-redis.ts
+++ b/apps/api/src/lib/deep-research/deep-research-redis.ts
@@ -1,15 +1,6 @@
import { redisEvictConnection } from "../../services/redis";
import { logger as _logger } from "../logger";
-export enum DeepResearchStep {
- INITIAL = "initial",
- SEARCH = "search",
- EXTRACT = "extract",
- ANALYZE = "analyze",
- SYNTHESIS = "synthesis",
- COMPLETE = "complete",
-}
-
export type DeepResearchActivity = {
type:
| "search"
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 6a22db560..bd76e17aa 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -1,16 +1,4 @@
-import type { Document as V1Document, Action } from "../controllers/v1/types";
-
-export interface Progress {
- current: number;
- total: number;
- status: string;
- metadata?: {
- sourceURL?: string;
- [key: string]: any;
- };
- currentDocumentUrl?: string;
- currentDocument?: Document;
-}
+import type { Action } from "../controllers/v1/types";
export type PageOptions = {
includeMarkdown?: boolean;
@@ -61,38 +49,6 @@ export type SearchOptions = {
location?: string;
};
-export type CrawlerOptions = {
- returnOnlyUrls?: boolean;
- includes?: string | string[];
- excludes?: string | string[];
- maxCrawledLinks?: number;
- maxDepth?: number;
- limit?: number;
- generateImgAltText?: boolean;
- replaceAllPathsWithAbsolutePaths?: boolean;
- ignoreSitemap?: boolean;
- mode?: "default" | "fast"; // have a mode of some sort
- allowBackwardCrawling?: boolean;
- allowExternalContentLinks?: boolean;
-};
-
-export type WebScraperOptions = {
- jobId: string;
- urls: string[];
- mode: "single_urls" | "sitemap" | "crawl";
- crawlerOptions?: CrawlerOptions;
- pageOptions?: PageOptions;
- extractorOptions?: ExtractorOptions;
- concurrentRequests?: number;
- bullJobId?: string;
- priority?: number;
- teamId?: string;
-};
-
-export interface DocumentUrl {
- url: string;
-}
-
export class Document {
id?: string;
url?: string; // Used only in /search for now
@@ -151,7 +107,7 @@ export class SearchResult {
}
}
-export interface ImageSearchResult {
+interface ImageSearchResult {
title?: string;
imageUrl?: string;
imageWidth?: number;
@@ -160,7 +116,7 @@ export interface ImageSearchResult {
position?: number;
}
-export interface NewsSearchResult {
+interface NewsSearchResult {
title?: string;
url?: string;
snippet?: string;
@@ -204,21 +160,3 @@ export interface ScrapeActionContent {
url: string;
html: string;
}
-
-export interface FireEngineResponse {
- html: string;
- screenshots?: string[];
- pageStatusCode?: number;
- pageError?: string;
- scrapeActionContent?: ScrapeActionContent[];
-}
-
-export interface FireEngineOptions {
- mobileProxy?: boolean;
- method?: string;
- engine?: string;
- blockMedia?: boolean;
- blockAds?: boolean;
- disableJsDom?: boolean;
- atsv?: boolean; // beta
-}
diff --git a/apps/api/src/lib/extract/build-prompts.ts b/apps/api/src/lib/extract/build-prompts.ts
index 655b87edb..a6b8d00ce 100644
--- a/apps/api/src/lib/extract/build-prompts.ts
+++ b/apps/api/src/lib/extract/build-prompts.ts
@@ -30,15 +30,6 @@ Website to get content from: ${url}
Return only a concise sentece or 2 focused on the essential data points that the user wants to extract. This will be used by an LLM to determine how releavant the links that are present are to the user's request.`;
}
-export function buildRerankerSystemPrompt(): string {
- return `You are a relevance expert scoring links from a website the user is trying to extract information from. Analyze the provided URLs and their content
-to determine their relevance to the user's query and intent.
- For each URL, assign a relevance score between 0 and 1, where 1
- means highly relevant and we should extract the content from it and 0 means not relevant at all, we should not extract the content from it.
- Always return all the links scored that you are giving. Do not omit links.
- Always return the links in the same order they were provided. If the user wants the content from all the links, all links should be scored 1.`;
-}
-
export function buildRerankerUserPrompt(searchQuery: string): string {
return `Given these URLs and their content, analyze their relevance to this extraction request: "${searchQuery}".
@@ -89,19 +80,6 @@ export function buildAnalyzeSchemaUserPrompt(
Schema: ${schemaString}\nPrompt: ${prompt}\n URLs: ${urls}`;
}
-// Should Extract
-
-export function buildShouldExtractSystemPrompt(): string {
- return `You are a content relevance checker. Your job is to determine if the provided content is very relevant to extract information from based on the user's prompt. Return true only if the content appears relevant and contains information that could help answer the prompt. Return false if the content seems irrelevant or unlikely to contain useful information for the prompt.`;
-}
-
-export function buildShouldExtractUserPrompt(
- prompt: string,
- schema: any,
-): string {
- return `Should the following content be used to extract information for this prompt: "${prompt}" User schema is: ${JSON.stringify(schema)}\nReturn only true or false.`;
-}
-
// Batch extract
export function buildBatchExtractSystemPrompt(
systemPrompt: string,
@@ -123,25 +101,3 @@ export function buildRephraseToSerpPrompt(prompt: string): string {
Original Prompt: "${prompt}"`;
}
-
-export function buildCrawlerOptionsPrompt(prompt: string): string {
- return `You are a web crawler configuration expert. Based on the user's natural language instruction, generate appropriate crawler options.
-
-User instruction: "${prompt}"
-
-Generate a JSON object with crawler configuration options. Available options:
-- includePaths: string[] - URL patterns to include (regex patterns)
-- excludePaths: string[] - URL patterns to exclude (regex patterns)
-- maxDepth: number - Maximum crawl depth (default: 10)
-- crawlEntireDomain: boolean - Whether to crawl the entire domain
-- allowExternalLinks: boolean - Whether to follow external links
-- allowSubdomains: boolean - Whether to crawl subdomains
-- ignoreRobotsTxt: boolean - Whether to ignore robots.txt
-- sitemap: "skip" | "include" - Whether to ignore sitemap
-- deduplicateSimilarURLs: boolean - Whether to deduplicate similar URLs
-- ignoreQueryParameters: boolean - Whether to ignore query parameters
-- regexOnFullURL: boolean - Whether regex patterns apply to full URL
-- delay: number - Delay between requests in milliseconds
-
-Return only a valid JSON object with the appropriate options based on the user's intent. If the instruction doesn't clearly map to specific options, use reasonable defaults.`;
-}
diff --git a/apps/api/src/lib/extract/completions/checkShouldExtract.ts b/apps/api/src/lib/extract/completions/checkShouldExtract.ts
deleted file mode 100644
index 7120a7675..000000000
--- a/apps/api/src/lib/extract/completions/checkShouldExtract.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-import { logger } from "../../../lib/logger";
-import { buildDocument } from "../build-document";
-import { Document, TokenUsage } from "../../../controllers/v1/types";
-import { generateCompletions } from "../../../scraper/scrapeURL/transformers/llmExtract";
-import {
- buildShouldExtractSystemPrompt,
- buildShouldExtractUserPrompt,
-} from "../build-prompts";
-import { getModel } from "../../../lib/generic-ai";
-import { CostTracking } from "../../cost-tracking";
-
-export async function checkShouldExtract(
- prompt: string,
- multiEntitySchema: any,
- doc: Document,
- costTracking: CostTracking,
- metadata: {
- teamId: string;
- functionId?: string;
- extractId?: string;
- scrapeId?: string;
- deepResearchId?: string;
- },
-): Promise<{ tokenUsage: TokenUsage; extract: boolean }> {
- const shouldExtractCheck = await generateCompletions({
- logger: logger.child({ method: "extractService/checkShouldExtract" }),
- options: {
- systemPrompt: buildShouldExtractSystemPrompt(),
- prompt: buildShouldExtractUserPrompt(prompt, multiEntitySchema),
- schema: {
- type: "object",
- properties: {
- extract: {
- type: "boolean",
- },
- },
- required: ["extract"],
- },
- },
- markdown: buildDocument(doc),
- isExtractEndpoint: true,
- model: getModel("gpt-4o-mini", "openai"),
- costTrackingOptions: {
- costTracking,
- metadata: {
- module: "extract",
- method: "checkShouldExtract",
- },
- },
- metadata: {
- ...metadata,
- functionId: metadata.functionId
- ? metadata.functionId + "/checkShouldExtract"
- : "checkShouldExtract",
- },
- });
-
- return {
- tokenUsage: shouldExtractCheck.totalUsage,
- extract: shouldExtractCheck.extract["extract"],
- };
-}
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts
index be9ac4a6b..a33a061d4 100644
--- a/apps/api/src/lib/extract/extract-redis.ts
+++ b/apps/api/src/lib/extract/extract-redis.ts
@@ -15,7 +15,7 @@ export enum ExtractStep {
COMPLETE = "complete",
}
-export type ExtractedStep = {
+type ExtractedStep = {
step: ExtractStep;
startedAt: number;
finishedAt: number | null;
@@ -23,7 +23,7 @@ export type ExtractedStep = {
discoveredLinks?: string[];
};
-export type StoredExtract = {
+type StoredExtract = {
id: string;
team_id: string;
createdAt: number;
diff --git a/apps/api/src/lib/extract/fire-0/build-prompts-f0.ts b/apps/api/src/lib/extract/fire-0/build-prompts-f0.ts
index 0e072a302..9fc3f3252 100644
--- a/apps/api/src/lib/extract/fire-0/build-prompts-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/build-prompts-f0.ts
@@ -1,35 +1,3 @@
-export function buildRefrasedPrompt_F0(prompt: string, url: string): string {
- return `You are a search query optimizer. Your task is to rephrase the following prompt into an effective search query that will find relevant results about this topic on ${url}.
-
- Original prompt: "${prompt}"
-
- Provide a rephrased search query that:
- 1. Maintains the core intent of the original prompt with ONLY the keywords
- 2. Uses relevant keywords
- 3. Is optimized for search engine results
- 4. Is concise and focused
- 5. Short is better than long
- 6. It is a search engine, not a chatbot
- 7. Concise
-
- Return only the rephrased search query, without any explanation or additional text.`;
-}
-
-export function buildPreRerankPrompt_F0(
- prompt: string | undefined,
- schema: any,
- url: string,
-): string {
- const schemaString = JSON.stringify(schema, null, 2);
- return `Create a concise search query that combines the key data points from both the schema and prompt. Focus on the core information needed while keeping it general enough to find relevant matches.
-
- Schema: ${schemaString}
- Prompt: ${prompt}
- Website to get content from: ${url}
-
- Return only a concise sentece or 2 focused on the essential data points that the user wants to extract. This will be used by an LLM to determine how releavant the links that are present are to the user's request.`;
-}
-
export function buildRerankerSystemPrompt_F0(): string {
return `You are a relevance expert scoring links from a website the user is trying to extract information from. Analyze the provided URLs and their content
to determine their relevance to the user's query and intent.
@@ -43,39 +11,6 @@ export function buildRerankerUserPrompt_F0(searchQuery: string): string {
return `Given these URLs and their content, identify which ones are relevant to the user's extraction request: "${searchQuery}". Return an array of relevant links with their relevance scores (0-1). Higher scores should be given to URLs that directly address the user's extraction request. Be very mindful with the links you select, as if they are not that relevant it may affect the quality of the extraction. Only include URLs that have a relevancy score of 0.6+.`;
}
-// Multi entity schema anlayzer
-export function buildAnalyzeSchemaPrompt_F0(): string {
- return `You are a query classifier for a web scraping system. Classify the data extraction query as either:
- A) Single-Answer: One answer across a few pages, possibly containing small arrays.
- B) Multi-Entity: Many items across many pages, often involving large arrays.
-
- Consider:
- 1. Answer Cardinality: Single or multiple items?
- 2. Page Distribution: Found on 1-3 pages or many?
- 3. Verification Needs: Cross-page verification or independent extraction?
-
- Provide:
- - Method: [Single-Answer/Multi-Entity]
- - Confidence: [0-100%]
- - Reasoning: Why this classification?
- - Key Indicators: Specific aspects leading to this decision.
-
- Examples:
- - "Is this company a non-profit?" -> Single-Answer
- - "Extract all product prices" -> Multi-Entity
-
- For Single-Answer, arrays may be present but are typically small. For Multi-Entity, if arrays have multiple items not from a single page, return keys with large arrays. If nested, return the full key (e.g., 'ecommerce.products').`;
-}
-
-export function buildAnalyzeSchemaUserPrompt_F0(
- schemaString: string,
- prompt: string,
- urls: string[],
-): string {
- return `Classify the query as Single-Answer or Multi-Entity. For Multi-Entity, return keys with large arrays; otherwise, return none:
- Schema: ${schemaString}\nPrompt: ${prompt}\nRelevant URLs: ${urls}`;
-}
-
// Should Extract
export function buildShouldExtractSystemPrompt_F0(): string {
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index c6a2d92be..8de1cb842 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -50,7 +50,7 @@ interface ExtractServiceOptions {
apiKeyId: number | null;
}
-export interface ExtractResult {
+interface ExtractResult {
success: boolean;
data?: any;
extractId: string;
diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
index 54b0d4a73..ecd761672 100644
--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
@@ -31,7 +31,7 @@ const getModelLimits_F0 = (model: string) => {
};
};
-export class LLMRefusalError extends Error {
+class LLMRefusalError extends Error {
public refusal: string;
constructor(refusal: string) {
@@ -96,7 +96,7 @@ interface TrimResult {
warning?: string;
}
-export function trimToTokenLimit_F0(
+function trimToTokenLimit_F0(
text: string,
maxTokens: number,
modelId: string = "gpt-4o",
@@ -446,7 +446,7 @@ export async function generateCompletions_F0({
}
}
-export function removeDefaultProperty_F0(schema: any): any {
+function removeDefaultProperty_F0(schema: any): any {
if (typeof schema !== "object" || schema === null) return schema;
const rest = { ...schema };
diff --git a/apps/api/src/lib/extract/fire-0/ranker-f0.ts b/apps/api/src/lib/extract/fire-0/ranker-f0.ts
index 470969343..0ef2eff8f 100644
--- a/apps/api/src/lib/extract/fire-0/ranker-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/ranker-f0.ts
@@ -98,5 +98,3 @@ async function performRanking_F0(
return [];
}
}
-
-export { performRanking_F0 };
diff --git a/apps/api/src/lib/extract/fire-0/reranker-f0.ts b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
index d9e8cb5c0..58e33e8fd 100644
--- a/apps/api/src/lib/extract/fire-0/reranker-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
@@ -1,181 +1,18 @@
-import {
- MapDocument,
- TeamFlags,
- URLTrace,
-} from "../../../controllers/v1/types";
-import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";
+import { MapDocument, URLTrace } from "../../../controllers/v1/types";
import { logger } from "../../logger";
-import { CohereClient } from "cohere-ai";
-import { extractConfig } from "../config";
import { generateCompletions } from "../../../scraper/scrapeURL/transformers/llmExtract";
-import { performRanking_F0 } from "./ranker-f0";
import {
buildRerankerSystemPrompt_F0,
buildRerankerUserPrompt_F0,
} from "./build-prompts-f0";
import { CostTracking } from "../../cost-tracking";
-const cohere = new CohereClient({
- token: process.env.COHERE_API_KEY,
-});
-
-interface RankingResult {
- mappedLinks: MapDocument[];
- linksAndScores: {
- link: string;
- linkWithContext: string;
- score: number;
- originalIndex: number;
- }[];
-}
-
-export async function rerankDocuments_FO(
- documents: (string | Record<string, string>)[],
- query: string,
- topN = 3,
- model = "rerank-english-v3.0",
-) {
- const rerank = await cohere.v2.rerank({
- documents,
- query,
- topN,
- model,
- returnDocuments: true,
- });
-
- return rerank.results
- .sort((a, b) => b.relevanceScore - a.relevanceScore)
- .map(x => ({
- document: x.document,
- index: x.index,
- relevanceScore: x.relevanceScore,
- }));
-}
-
-export async function rerankLinks_F0(
- mappedLinks: MapDocument[],
- searchQuery: string,
- urlTraces: URLTrace[],
- flags: TeamFlags,
- metadata: { teamId: string; extractId?: string },
-): Promise<MapDocument[]> {
- // console.log("Going to rerank links");
- const mappedLinksRerank = mappedLinks.map(
- x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`,
- );
-
- const linksAndScores = await performRanking_F0(
- mappedLinksRerank,
- mappedLinks.map(l => l.url),
- searchQuery,
- metadata,
- );
-
- // First try with high threshold
- let filteredLinks = filterAndProcessLinks_F0(
- mappedLinks,
- linksAndScores,
- extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE,
- flags,
- );
-
- // If we don't have enough high-quality links, try with lower threshold
- if (filteredLinks.length < extractConfig.RERANKING.MIN_REQUIRED_LINKS) {
- logger.info(
- `Only found ${filteredLinks.length} links with score > ${extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE}. Trying lower threshold...`,
- );
- filteredLinks = filterAndProcessLinks_F0(
- mappedLinks,
- linksAndScores,
- extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE,
- flags,
- );
-
- if (filteredLinks.length === 0) {
- // If still no results, take top N results regardless of score
- logger.warn(
- `No links found with score > ${extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE}. Taking top ${extractConfig.RERANKING.MIN_REQUIRED_LINKS} results.`,
- );
- filteredLinks = linksAndScores
- .sort((a, b) => b.score - a.score)
- .slice(0, extractConfig.RERANKING.MIN_REQUIRED_LINKS)
- .map(x => mappedLinks.find(link => link.url === x.link))
- .filter(
- (x): x is MapDocument =>
- x !== undefined &&
- x.url !== undefined &&
- !isUrlBlocked(x.url, flags),
- );
- }
- }
-
- // Update URL traces with relevance scores and mark filtered out URLs
- linksAndScores.forEach(score => {
- const trace = urlTraces.find(t => t.url === score.link);
- if (trace) {
- trace.relevanceScore = score.score;
- // If URL didn't make it through filtering, mark it as filtered out
- if (!filteredLinks.some(link => link.url === score.link)) {
- trace.warning = `Relevance score ${score.score} below threshold`;
- trace.usedInCompletion = false;
- }
- }
- });
-
- const rankedLinks = filteredLinks.slice(
- 0,
- extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE,
- );
-
- // Mark URLs that will be used in completion
- rankedLinks.forEach(link => {
- const trace = urlTraces.find(t => t.url === link.url);
- if (trace) {
- trace.usedInCompletion = true;
- }
- });
-
- // Mark URLs that were dropped due to ranking limit
- filteredLinks
- .slice(extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE)
- .forEach(link => {
- const trace = urlTraces.find(t => t.url === link.url);
- if (trace) {
- trace.warning = "Excluded due to ranking limit";
- trace.usedInCompletion = false;
- }
- });
-
- // console.log("Reranked links: ", rankedLinks.length);
-
- return rankedLinks;
-}
-
-function filterAndProcessLinks_F0(
- mappedLinks: MapDocument[],
- linksAndScores: {
- link: string;
- linkWithContext: string;
- score: number;
- originalIndex: number;
- }[],
- threshold: number,
- flags: TeamFlags,
-): MapDocument[] {
- return linksAndScores
- .filter(x => x.score > threshold)
- .map(x => mappedLinks.find(link => link.url === x.link))
- .filter(
- (x): x is MapDocument =>
- x !== undefined && x.url !== undefined && !isUrlBlocked(x.url, flags),
- );
-}
-export type RerankerResult = {
+type RerankerResult = {
mapDocument: (MapDocument & { relevanceScore?: number; reason?: string })[];
tokensUsed: number;
};
-export type RerankerOptions = {
+type RerankerOptions = {
links: MapDocument[];
searchQuery: string;
urlTraces: URLTrace[];
diff --git a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts
index ab6db63f6..b41a021f5 100644
--- a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts
@@ -23,7 +23,7 @@ export function estimateTotalCost_F0(tokenUsage: TokenUsage[]): number {
}, 0);
}
-export function estimateCost_F0(tokenUsage: TokenUsage): number {
+function estimateCost_F0(tokenUsage: TokenUsage): number {
let totalCost = 0;
try {
let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");
diff --git a/apps/api/src/lib/extract/helpers/dump-to-file.ts b/apps/api/src/lib/extract/helpers/dump-to-file.ts
deleted file mode 100644
index f52b924c1..000000000
--- a/apps/api/src/lib/extract/helpers/dump-to-file.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import * as fs from "fs";
-import * as path from "path";
-
-/**
- * Helper function to dump data to a file for debugging/logging purposes
- * @param filename The name of the file to write to (will be created in __dirname)
- * @param data The data to write to the file
- * @param formatter Optional function to format each item in the data array
- */
-export function dumpToFile<T>(
- filename: string,
- data: T[],
- formatter?: (item: T, index: number) => string,
-) {
- const filePath = path.join(__dirname, filename);
-
- let fileContent: string;
- if (formatter) {
- fileContent = data.map((item, index) => formatter(item, index)).join("\n");
- } else {
- fileContent = data
- .map((item, index) => `${index + 1}. ${JSON.stringify(item)}`)
- .join("\n");
- }
-
- fs.writeFileSync(filePath, fileContent, "utf8");
- console.log(`Dumped data to ${filename}`);
-}
diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts
index c06949ab4..44a1b8b30 100644
--- a/apps/api/src/lib/extract/reranker.ts
+++ b/apps/api/src/lib/extract/reranker.ts
@@ -1,183 +1,20 @@
-import { MapDocument, TeamFlags, URLTrace } from "../../controllers/v1/types";
-import { performRanking } from "../ranker";
-import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { MapDocument, URLTrace } from "../../controllers/v1/types";
import { logger } from "../logger";
-import { CohereClient } from "cohere-ai";
-import { extractConfig } from "./config";
import { generateCompletions } from "../../scraper/scrapeURL/transformers/llmExtract";
import { buildRerankerUserPrompt } from "./build-prompts";
-import { buildRerankerSystemPrompt } from "./build-prompts";
-import { dumpToFile } from "./helpers/dump-to-file";
import { getModel } from "../generic-ai";
-import fs from "fs/promises";
import { CostTracking } from "../cost-tracking";
const THRESHOLD_FOR_SINGLEPAGE = 0.6;
const THRESHOLD_FOR_MULTIENTITY = 0.45;
-const cohere = new CohereClient({
- token: process.env.COHERE_API_KEY,
-});
-
-interface RankingResult {
- mappedLinks: MapDocument[];
- linksAndScores: {
- link: string;
- linkWithContext: string;
- score: number;
- originalIndex: number;
- }[];
-}
-
-export async function rerankDocuments(
- documents: (string | Record<string, string>)[],
- query: string,
- topN = 3,
- model = "rerank-english-v3.0",
-) {
- const rerank = await cohere.v2.rerank({
- documents,
- query,
- topN,
- model,
- returnDocuments: true,
- });
-
- return rerank.results
- .sort((a, b) => b.relevanceScore - a.relevanceScore)
- .map(x => ({
- document: x.document,
- index: x.index,
- relevanceScore: x.relevanceScore,
- }));
-}
-
-export async function rerankLinks(
- mappedLinks: MapDocument[],
- searchQuery: string,
- urlTraces: URLTrace[],
- flags: TeamFlags,
- metadata: { teamId: string; extractId?: string },
-): Promise<MapDocument[]> {
- // console.log("Going to rerank links");
- const mappedLinksRerank = mappedLinks.map(
- x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`,
- );
-
- const linksAndScores = await performRanking(
- mappedLinksRerank,
- mappedLinks.map(l => l.url),
- searchQuery,
- metadata,
- );
-
- // First try with high threshold
- let filteredLinks = filterAndProcessLinks(
- mappedLinks,
- linksAndScores,
- extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE,
- flags,
- );
-
- // If we don't have enough high-quality links, try with lower threshold
- if (filteredLinks.length < extractConfig.RERANKING.MIN_REQUIRED_LINKS) {
- logger.info(
- `Only found ${filteredLinks.length} links with score > ${extractConfig.RERANKING.INITIAL_SCORE_THRESHOLD_FOR_RELEVANCE}. Trying lower threshold...`,
- );
- filteredLinks = filterAndProcessLinks(
- mappedLinks,
- linksAndScores,
- extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE,
- flags,
- );
-
- if (filteredLinks.length === 0) {
- // If still no results, take top N results regardless of score
- logger.warn(
- `No links found with score > ${extractConfig.RERANKING.FALLBACK_SCORE_THRESHOLD_FOR_RELEVANCE}. Taking top ${extractConfig.RERANKING.MIN_REQUIRED_LINKS} results.`,
- );
- filteredLinks = linksAndScores
- .sort((a, b) => b.score - a.score)
- .slice(0, extractConfig.RERANKING.MIN_REQUIRED_LINKS)
- .map(x => mappedLinks.find(link => link.url === x.link))
- .filter(
- (x): x is MapDocument =>
- x !== undefined &&
- x.url !== undefined &&
- !isUrlBlocked(x.url, flags),
- );
- }
- }
-
- // Update URL traces with relevance scores and mark filtered out URLs
- linksAndScores.forEach(score => {
- const trace = urlTraces.find(t => t.url === score.link);
- if (trace) {
- trace.relevanceScore = score.score;
- // If URL didn't make it through filtering, mark it as filtered out
- if (!filteredLinks.some(link => link.url === score.link)) {
- trace.warning = `Relevance score ${score.score} below threshold`;
- trace.usedInCompletion = false;
- }
- }
- });
-
- const rankedLinks = filteredLinks.slice(
- 0,
- extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE,
- );
-
- // Mark URLs that will be used in completion
- rankedLinks.forEach(link => {
- const trace = urlTraces.find(t => t.url === link.url);
- if (trace) {
- trace.usedInCompletion = true;
- }
- });
-
- // Mark URLs that were dropped due to ranking limit
- filteredLinks
- .slice(extractConfig.RERANKING.MAX_RANKING_LIMIT_FOR_RELEVANCE)
- .forEach(link => {
- const trace = urlTraces.find(t => t.url === link.url);
- if (trace) {
- trace.warning = "Excluded due to ranking limit";
- trace.usedInCompletion = false;
- }
- });
-
- // console.log("Reranked links: ", rankedLinks.length);
-
- return rankedLinks;
-}
-
-function filterAndProcessLinks(
- mappedLinks: MapDocument[],
- linksAndScores: {
- link: string;
- linkWithContext: string;
- score: number;
- originalIndex: number;
- }[],
- threshold: number,
- flags: TeamFlags,
-): MapDocument[] {
- return linksAndScores
- .filter(x => x.score > threshold)
- .map(x => mappedLinks.find(link => link.url === x.link))
- .filter(
- (x): x is MapDocument =>
- x !== undefined && x.url !== undefined && !isUrlBlocked(x.url, flags),
- );
-}
-
-export type RerankerResult = {
+type RerankerResult = {
mapDocument: (MapDocument & { relevanceScore?: number; reason?: string })[];
tokensUsed: number;
cost: number;
};
-export type RerankerOptions = {
+type RerankerOptions = {
links: MapDocument[];
searchQuery: string;
urlTraces: URLTrace[];
diff --git a/apps/api/src/lib/extract/usage/llm-cost.ts b/apps/api/src/lib/extract/usage/llm-cost.ts
index 4e675acef..4a9fd3a95 100644
--- a/apps/api/src/lib/extract/usage/llm-cost.ts
+++ b/apps/api/src/lib/extract/usage/llm-cost.ts
@@ -28,7 +28,7 @@ export function estimateTotalCost(tokenUsage: TokenUsage[]): number {
}, 0);
}
-export function estimateCost(tokenUsage: TokenUsage): number {
+function estimateCost(tokenUsage: TokenUsage): number {
let totalCost = 0;
try {
let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");
diff --git a/apps/api/src/lib/format-utils.ts b/apps/api/src/lib/format-utils.ts
index fa438be81..119fbd2b9 100644
--- a/apps/api/src/lib/format-utils.ts
+++ b/apps/api/src/lib/format-utils.ts
@@ -41,42 +41,3 @@ export function hasAnyFormatOfTypes(
return formats.some(f => types.includes(f.type));
}
-
-/**
- * Gets all format objects of a specific type from the formats array.
- * This is useful when multiple formats of the same type might exist.
- *
- * @param formats - Array of format objects
- * @param type - The format type to filter by
- * @returns Array of format objects matching the type
- */
-export function getFormatsOfType<T extends FormatObject["type"]>(
- formats: FormatObject[] | undefined,
- type: T,
-): Extract<FormatObject, { type: T }>[] {
- if (!formats) {
- return [];
- }
-
- return formats.filter(f => f.type === type) as Extract<
- FormatObject,
- { type: T }
- >[];
-}
-
-/**
- * Extracts just the format types from an array of format objects.
- * Useful for checking which formats are present without their options.
- *
- * @param formats - Array of format objects
- * @returns Array of format type strings
- */
-export function getFormatTypes(
- formats: FormatObject[] | undefined,
-): FormatObject["type"][] {
- if (!formats) {
- return [];
- }
-
- return formats.map(f => f.type);
-}
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 70eaa099e..5f42a5734 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -11,7 +11,7 @@ const PDF_CACHE_PREFIX = "pdf-cache-v2/";
* Creates a SHA-256 hash of the PDF content to use as a cache key
* Directly hashes the content without any conversion
*/
-export function createPdfCacheKey(pdfContent: string | Buffer): string {
+function createPdfCacheKey(pdfContent: string | Buffer): string {
return crypto.createHash("sha256").update(pdfContent).digest("hex");
}
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts
index 3a7f1aaee..8eab4e4aa 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts
@@ -1,7 +1,7 @@
import { redisEvictConnection } from "../../services/redis";
import { logger as _logger } from "../logger";
-export interface GenerationData {
+interface GenerationData {
id: string;
team_id: string;
createdAt: number;
@@ -62,7 +62,7 @@ export async function getGeneratedLlmsTxtExpiry(id: string): Promise<Date> {
}
// Convenience method for status updates
-export async function updateGeneratedLlmsTxtStatus(
+async function updateGeneratedLlmsTxtStatus(
id: string,
status: "processing" | "completed" | "failed",
generatedText?: string,
diff --git a/apps/api/src/lib/parse-mode.ts b/apps/api/src/lib/parse-mode.ts
deleted file mode 100644
index 91c04ef14..000000000
--- a/apps/api/src/lib/parse-mode.ts
+++ /dev/null
@@ -1,12 +0,0 @@
-export function parseMode(mode: string) {
- switch (mode) {
- case "single_urls":
- return "single_urls";
- case "sitemap":
- return "sitemap";
- case "crawl":
- return "crawl";
- default:
- return "single_urls";
- }
-}
diff --git a/apps/api/src/lib/parseApi.ts b/apps/api/src/lib/parseApi.ts
index e29135fd3..4479d44d1 100644
--- a/apps/api/src/lib/parseApi.ts
+++ b/apps/api/src/lib/parseApi.ts
@@ -12,8 +12,3 @@ export function parseApi(api: string) {
.replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, "$1-$2-$3-$4-$5");
return uuid;
}
-
-export function uuidToFcUuid(uuid: string) {
- const uuidWithoutDashes = uuid.replace(/-/g, "");
- return `fc-${uuidWithoutDashes}`;
-}
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 656299e39..1a67051d2 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -9,7 +9,7 @@ const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
process.env.FIRE_ENGINE_BETA_URL !== undefined;
-export interface RobotsTxtChecker {
+interface RobotsTxtChecker {
robotsTxtUrl: string;
robotsTxt: string;
robots: Robot;
diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.ts
index ec1bbe339..613d0fd86 100644
--- a/apps/api/src/lib/search-query-builder.ts
+++ b/apps/api/src/lib/search-query-builder.ts
@@ -3,14 +3,14 @@
* Builds search queries with category filters for the search API
*/
-export interface CategoryInput {
+interface CategoryInput {
type: "github" | "research";
sites?: string[];
}
export type CategoryOption = string | CategoryInput;
-export interface QueryBuilderResult {
+interface QueryBuilderResult {
query: string;
categoryMap: Map<string, string>;
}
diff --git a/apps/api/src/lib/timeout.ts b/apps/api/src/lib/timeout.ts
deleted file mode 100644
index f913817a1..000000000
--- a/apps/api/src/lib/timeout.ts
+++ /dev/null
@@ -1 +0,0 @@
-export const axiosTimeout = 5000;
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 7d90faffe..0ef187f87 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -42,7 +42,7 @@ export async function startWebScraperPipeline({
});
}
-export async function runWebScraper({
+async function runWebScraper({
url,
mode,
scrapeOptions,
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index be91be585..e34335134 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -20,13 +20,13 @@ import { filterLinks } from "@mendable/firecrawl-rs";
export const SITEMAP_LIMIT = 100;
-export interface FilterResult {
+interface FilterResult {
allowed: boolean;
url?: string;
denialReason?: string;
}
-export enum DenialReason {
+enum DenialReason {
DEPTH_LIMIT = "URL exceeds maximum crawl depth",
EXCLUDE_PATTERN = "URL matches exclude pattern",
INCLUDE_PATTERN = "URL does not match required include pattern",
@@ -39,7 +39,7 @@ export enum DenialReason {
SECTION_LINK = "URL contains section anchor (#)",
}
-export interface FilterLinksResult {
+interface FilterLinksResult {
links: string[];
denialReasons: Map<string, string>;
}
diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
deleted file mode 100644
index 01c40de9b..000000000
--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
+++ /dev/null
@@ -1,67 +0,0 @@
-import { logger } from "../../../lib/logger";
-
-export async function handleCustomScraping(
- text: string,
- url: string,
-): Promise<{
- scraper: string;
- url: string;
- waitAfterLoad?: number;
- pageOptions?: { scrollXPaths?: string[] };
-} | null> {
- // Check for Readme Docs special case
- if (
- text.includes('<meta name="readme-deploy"') &&
- !url.includes("developers.notion.com")
- ) {
- logger.debug(
- `Special use case detected for ${url}, using Fire Engine with wait time 1000ms`,
- );
- return {
- scraper: "fire-engine",
- url: url,
- waitAfterLoad: 1000,
- pageOptions: {
- scrollXPaths: [
- '//*[@id="ReferencePlayground"]/section[3]/div/pre/div/div/div[5]',
- ],
- },
- };
- }
-
- // Check for Vanta security portals
- if (text.includes('<link href="https://static.vanta.com')) {
- logger.debug(
- `Vanta link detected for ${url}, using Fire Engine with wait time 3000ms`,
- );
- return {
- scraper: "fire-engine",
- url: url,
- waitAfterLoad: 3000,
- };
- }
-
- // Check for Google Drive PDF links in meta tags
- const googleDriveMetaPattern =
- /<meta itemprop="url" content="(https:\/\/drive\.google\.com\/file\/d\/[^"]+)"/;
- const googleDriveMetaMatch = text.match(googleDriveMetaPattern);
- if (googleDriveMetaMatch) {
- const url = googleDriveMetaMatch[1];
- logger.debug(`Google Drive PDF link detected: ${url}`);
-
- const fileIdMatch = url.match(
- /https:\/\/drive\.google\.com\/file\/d\/([^\/]+)\/view/,
- );
- if (fileIdMatch) {
- const fileId = fileIdMatch[1];
- const pdfUrl = `https://drive.google.com/uc?export=download&id=${fileId}`;
-
- return {
- scraper: "pdf",
- url: pdfUrl,
- };
- }
- }
-
- return null;
-}
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 1299b6353..a69bf2404 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -8,15 +8,6 @@ configDotenv();
const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8");
const algorithm = "aes-256-ecb";
-export function encryptAES(plaintext: string, key: Buffer): string {
- const cipher = crypto.createCipheriv(algorithm, key, null);
- const encrypted = Buffer.concat([
- cipher.update(plaintext, "utf-8"),
- cipher.final(),
- ]);
- return encrypted.toString("base64");
-}
-
export function decryptAES(ciphertext: string, key: Buffer): string {
const decipher = crypto.createDecipheriv(algorithm, key, null);
const decrypted = Buffer.concat([
@@ -102,7 +93,7 @@ const allowedKeywords = [
"://www.aa.org",
];
-export function decryptedBlocklist(list: string[]): string[] {
+function decryptedBlocklist(list: string[]): string[] {
return hashKey.length > 0
? list.map(ciphertext => decryptAES(ciphertext, hashKey))
: [];
diff --git a/apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts b/apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts
deleted file mode 100644
index 73452c428..000000000
--- a/apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-export const removeBase64Images = async (markdown: string) => {
- const regex = /(!\[.*?\])\(data:image\/.*?;base64,.*?\)/g;
- markdown = markdown.replace(regex, "$1(<Base64-Image-Removed>)");
- return markdown;
-};
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index fc67828ea..93b4e429a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -152,7 +152,7 @@ const successSchema = z.object({
usedMobileProxy: z.boolean().optional(),
});
-export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;
+type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;
const processingSchema = z.object({
jobId: z.string(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 24028270d..246e27a7e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -42,7 +42,7 @@ const usePlaywright =
process.env.PLAYWRIGHT_MICROSERVICE_URL !== "" &&
process.env.PLAYWRIGHT_MICROSERVICE_URL !== undefined;
-export const engines: Engine[] = [
+const engines: Engine[] = [
...(useIndex ? ["index" as const, "index;documents" as const] : []),
...(useFireEngine
? [
@@ -62,7 +62,7 @@ export const engines: Engine[] = [
"docx",
];
-export const featureFlags = [
+const featureFlags = [
"actions",
"waitFor",
"screenshot",
@@ -80,7 +80,7 @@ export const featureFlags = [
export type FeatureFlag = (typeof featureFlags)[number];
-export const featureFlagOptions: {
+const featureFlagOptions: {
[F in FeatureFlag]: {
priority: number;
};
@@ -176,7 +176,7 @@ const engineMRTs: {
docx: docxMaxReasonableTime,
};
-export const engineOptions: {
+const engineOptions: {
[E in Engine]: {
// A list of feature flags the engine supports.
features: { [F in FeatureFlag]: boolean };
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index cad336b43..031e3beb3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -18,7 +18,7 @@ export function isIPPrivate(address: string): boolean {
return addr.range() !== "unicast";
}
-export function makeSecureDispatcher(skipTlsVerification: boolean) {
+function makeSecureDispatcher(skipTlsVerification: boolean) {
const agentOpts: undici.Agent.Options = {
maxRedirections: 5000,
};
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 2be357363..e2937ae44 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -292,7 +292,7 @@ export type InternalOptions = {
v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility
};
-export type EngineScrapeResultWithContext = {
+type EngineScrapeResultWithContext = {
engine: Engine;
unsupportedFeatures: Set<FeatureFlag>;
result: EngineScrapeResult;
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts
index 3b6d75e89..d909db311 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts
@@ -2,13 +2,13 @@ import { load } from "cheerio";
import { logger } from "../../../lib/logger";
import { extractAttributes as _extractAttributes } from "@mendable/firecrawl-rs";
-export type AttributeResult = {
+type AttributeResult = {
selector: string;
attribute: string;
values: string[];
};
-export type AttributeSelector = {
+type AttributeSelector = {
selector: string;
attribute: string;
};
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
index 9a891122e..1d4bc253f 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
@@ -3,7 +3,7 @@ import { Document } from "../../../controllers/v1/types";
import { Meta } from "..";
import { extractMetadata as _extractMetadata } from "@mendable/firecrawl-rs";
-export async function extractMetadataRust(
+async function extractMetadataRust(
meta: Meta,
html: string,
): Promise<Partial<Document["metadata"]>> {
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index e04a93f46..3088a46c2 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -81,69 +81,6 @@ const multiSmartScrapeWrapperSchemaDefinition = {
required: ["extractedData", "shouldUseSmartscrape"],
};
-//TODO: go over and check
-// should add null to all types
-// type:string should be type:["string","null"]
-export function makeSchemaNullable(schema: any): any {
- if (typeof schema !== "object" || schema === null) {
- return schema; // Base case: not an object/array or is null
- }
-
- if (Array.isArray(schema)) {
- return schema.map(makeSchemaNullable); // Recurse for array items
- }
-
- // Process object properties
- const newSchema: { [key: string]: any } = {};
- let isObject = false; // Flag to track if this level is an object type
-
- for (const key in schema) {
- if (key === "additionalProperties") {
- continue; // Skip existing additionalProperties, we'll set it later if needed
- }
-
- if (key === "type") {
- const currentType = schema[key];
- let finalType: string | string[];
-
- if (typeof currentType === "string") {
- if (currentType === "object") isObject = true;
- finalType =
- currentType === "null" ? currentType : [currentType, "null"];
- } else if (Array.isArray(currentType)) {
- if (currentType.includes("object")) isObject = true;
- finalType = currentType.includes("null")
- ? currentType
- : [...currentType, "null"];
- } else {
- finalType = currentType; // Handle unexpected types?
- }
- newSchema[key] = finalType;
- } else if (typeof schema[key] === "object" && schema[key] !== null) {
- // Recurse for nested objects (properties, items, definitions, etc.)
- newSchema[key] = makeSchemaNullable(schema[key]);
- if (key === "properties") {
- // Having a 'properties' key strongly implies an object type
- isObject = true;
- }
- } else {
- // Copy other properties directly (like required, description, etc.)
- newSchema[key] = schema[key];
- }
- }
-
- // **Crucial Fix:** If this schema represents an object type, add additionalProperties: false
- if (isObject) {
- // Ensure 'properties' exists if 'type' was 'object' but 'properties' wasn't defined
- if (!newSchema.properties) {
- newSchema.properties = {};
- }
- newSchema.additionalProperties = false;
- }
-
- return newSchema;
-}
-
/**
* Wraps the original schema with SmartScrape fields if an original schema exists.
*
@@ -151,7 +88,7 @@ export function makeSchemaNullable(schema: any): any {
* @param logger Winston logger instance.
* @returns An object containing the schema to use for the LLM call and whether wrapping occurred.
*/
-export function prepareSmartScrapeSchema(
+function prepareSmartScrapeSchema(
originalSchema: any | z.ZodTypeAny | undefined,
logger: Logger,
isSingleUrl: boolean,
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index 04022a701..da9866a05 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -8,7 +8,7 @@ import { cacheableLookup } from "./cacheableLookup";
import dns from "dns";
import { AbortManagerThrownError } from "./abortManager";
-export type RobustFetchParams<Schema extends z.Schema<any>> = {
+type RobustFetchParams<Schema extends z.Schema<any>> = {
url: string;
logger: Logger;
method: "GET" | "POST" | "DELETE" | "PUT";
diff --git a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts
index 8a3d6c3e5..fca2526e2 100644
--- a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts
@@ -1,7 +1,7 @@
import { InternalOptions } from "..";
import { ScrapeOptions } from "../../../controllers/v1/types";
-export type UrlSpecificParams = {
+type UrlSpecificParams = {
scrapeOptions: Partial<ScrapeOptions>;
internalOptions: Partial<InternalOptions>;
};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 06bf44c4f..e45b24a51 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -19,12 +19,12 @@ import {
hasAnyFormatOfTypes,
} from "../../../lib/format-utils";
-export type Transformer = (
+type Transformer = (
meta: Meta,
document: Document,
) => Document | Promise<Document>;
-export async function deriveMetadataFromRawHTML(
+async function deriveMetadataFromRawHTML(
meta: Meta,
document: Document,
): Promise<Document> {
@@ -41,7 +41,7 @@ export async function deriveMetadataFromRawHTML(
return document;
}
-export async function deriveHTMLFromRawHTML(
+async function deriveHTMLFromRawHTML(
meta: Meta,
document: Document,
): Promise<Document> {
@@ -62,7 +62,7 @@ export async function deriveHTMLFromRawHTML(
return document;
}
-export async function deriveMarkdownFromHTML(
+async function deriveMarkdownFromHTML(
meta: Meta,
document: Document,
): Promise<Document> {
@@ -112,7 +112,7 @@ export async function deriveMarkdownFromHTML(
return document;
}
-export async function deriveLinksFromHTML(
+async function deriveLinksFromHTML(
meta: Meta,
document: Document,
): Promise<Document> {
@@ -136,7 +136,7 @@ export async function deriveLinksFromHTML(
return document;
}
-export async function deriveImagesFromHTML(
+async function deriveImagesFromHTML(
meta: Meta,
document: Document,
): Promise<Document> {
@@ -160,10 +160,7 @@ export async function deriveImagesFromHTML(
return document;
}
-export function coerceFieldsToFormats(
- meta: Meta,
- document: Document,
-): Document {
+function coerceFieldsToFormats(meta: Meta, document: Document): Document {
const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");
const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");
const hasHtml = hasFormatOfType(meta.options.formats, "html");
@@ -331,7 +328,7 @@ export function coerceFieldsToFormats(
}
// TODO: allow some of these to run in parallel
-export const transformerStack: Transformer[] = [
+const transformerStack: Transformer[] = [
deriveHTMLFromRawHTML,
deriveMarkdownFromHTML,
deriveLinksFromHTML,
diff --git a/apps/api/src/search/googlesearch.ts b/apps/api/src/search/googlesearch.ts
index a43883e66..2344b7786 100644
--- a/apps/api/src/search/googlesearch.ts
+++ b/apps/api/src/search/googlesearch.ts
@@ -7,7 +7,7 @@ import https from "https";
const getRandomInt = (min: number, max: number): number =>
Math.floor(Math.random() * (max - min + 1)) + min;
-export function get_useragent(): string {
+function get_useragent(): string {
const lynx_version = `Lynx/${getRandomInt(2, 3)}.${getRandomInt(8, 9)}.${getRandomInt(0, 2)}`;
const libwww_version = `libwww-FM/${getRandomInt(2, 3)}.${getRandomInt(13, 15)}`;
const ssl_mm_version = `SSL-MM/${getRandomInt(1, 2)}.${getRandomInt(3, 5)}`;
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 351209d46..32417376c 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -64,86 +64,3 @@ export async function fire_engine_search_v2(
return {};
}
}
-
-// Backward compatibility function
-export async function fire_engine_search(
- q: string,
- options: {
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- numResults: number;
- page?: number;
- type?: SearchResultType | SearchResultType[];
- },
- abort?: AbortSignal,
-): Promise<SearchResult[]> {
- const result = await fire_engine_search_v2(q, options, abort);
-
- // Handle backward compatibility - convert to SearchResult array
- if (Array.isArray(result)) {
- return result;
- } else if (result.web) {
- return result.web.map(
- item => new SearchResult(item.url, item.title, item.description),
- );
- }
-
- return [];
-}
-
-export async function fireEngineMap(
- q: string,
- options: {
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- numResults: number;
- page?: number;
- },
- abort?: AbortSignal,
-): Promise<SearchResult[]> {
- try {
- let data = JSON.stringify({
- query: q,
- lang: options.lang,
- country: options.country,
- location: options.location,
- tbs: options.tbs,
- numResults: options.numResults,
- page: options.page ?? 1,
- });
-
- if (!process.env.FIRE_ENGINE_BETA_URL) {
- logger.warn(
- "(v1/map Beta) Results might differ from cloud offering currently.",
- );
- return [];
- }
-
- const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/map`, {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- "X-Disable-Cache": "true",
- },
- body: data,
- signal: abort,
- });
-
- if (response.ok) {
- const responseData = await response.json();
- return responseData;
- } else {
- return [];
- }
- } catch (error) {
- logger.error(error);
- Sentry.captureException(error);
- return [];
- }
-}
diff --git a/apps/api/src/search/v2/googlesearch.ts b/apps/api/src/search/v2/googlesearch.ts
index 9b74c33e4..76de00389 100644
--- a/apps/api/src/search/v2/googlesearch.ts
+++ b/apps/api/src/search/v2/googlesearch.ts
@@ -7,7 +7,7 @@ import https from "https";
const getRandomInt = (min: number, max: number): number =>
Math.floor(Math.random() * (max - min + 1)) + min;
-export function get_useragent(): string {
+function get_useragent(): string {
const lynx_version = `Lynx/${getRandomInt(2, 3)}.${getRandomInt(8, 9)}.${getRandomInt(0, 2)}`;
const libwww_version = `libwww-FM/${getRandomInt(2, 3)}.${getRandomInt(13, 15)}`;
const ssl_mm_version = `SSL-MM/${getRandomInt(1, 2)}.${getRandomInt(3, 5)}`;
diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts
index 0b616e335..76f408fdd 100644
--- a/apps/api/src/services/billing/credit_billing.ts
+++ b/apps/api/src/services/billing/credit_billing.ts
@@ -47,45 +47,7 @@ export async function billTeam(
)(team_id, subscription_id, credits, api_key_id, logger, is_extract);
}
-export async function supaBillTeam(
- team_id: string,
- subscription_id: string | null | undefined,
- credits: number,
- api_key_id: number | null,
- __logger?: Logger,
- is_extract: boolean = false,
-) {
- // This function should no longer be called directly
- // It has been moved to batch_billing.ts
- const _logger = (__logger ?? logger).child({
- module: "credit_billing",
- method: "supaBillTeam",
- teamId: team_id,
- subscriptionId: subscription_id,
- credits,
- });
-
- _logger.warn(
- "supaBillTeam was called directly. This function is deprecated and should only be called from batch_billing.ts",
- );
- queueBillingOperation(
- team_id,
- subscription_id,
- credits,
- api_key_id,
- is_extract,
- ).catch(err => {
- _logger.error("Error queuing billing operation", { err });
- Sentry.captureException(err);
- });
- // Forward to the batch billing system
- return {
- success: true,
- message: "Billing operation queued",
- };
-}
-
-export type CheckTeamCreditsResponse = {
+type CheckTeamCreditsResponse = {
success: boolean;
message: string;
remainingCredits: number;
@@ -105,7 +67,7 @@ export async function checkTeamCredits(
}
// if team has enough credits for the operation, return true, else return false
-export async function supaCheckTeamCredits(
+async function supaCheckTeamCredits(
chunk: AuthCreditUsageChunk | null,
team_id: string,
credits: number,
@@ -227,96 +189,3 @@ export async function supaCheckTeamCredits(
chunk,
};
}
-
-// Count the total credits used by a team within the current billing period and return the remaining credits.
-export async function countCreditsAndRemainingForCurrentBillingPeriod(
- team_id: string,
-) {
- // 1. Retrieve the team's active subscription based on the team_id.
- const { data: subscription, error: subscriptionError } =
- await supabase_service
- .from("subscriptions")
- .select("id, price_id, current_period_start, current_period_end")
- .eq("team_id", team_id)
- .single();
-
- const { data: coupons } = await supabase_service
- .from("coupons")
- .select("credits")
- .eq("team_id", team_id)
- .eq("status", "active");
-
- let couponCredits = 0;
- if (coupons && coupons.length > 0) {
- couponCredits = coupons.reduce(
- (total, coupon) => total + coupon.credits,
- 0,
- );
- }
-
- if (subscriptionError || !subscription) {
- // Free
- const { data: creditUsages, error: creditUsageError } =
- await supabase_service
- .from("credit_usage")
- .select("credits_used")
- .is("subscription_id", null)
- .eq("team_id", team_id);
-
- if (creditUsageError || !creditUsages) {
- throw new Error(
- `Failed to retrieve credit usage for team_id: ${team_id}`,
- );
- }
-
- const totalCreditsUsed = creditUsages.reduce(
- (acc, usage) => acc + usage.credits_used,
- 0,
- );
-
- const remainingCredits = FREE_CREDITS + couponCredits - totalCreditsUsed;
- return {
- totalCreditsUsed: totalCreditsUsed,
- remainingCredits,
- totalCredits: FREE_CREDITS + couponCredits,
- };
- }
-
- const { data: creditUsages, error: creditUsageError } = await supabase_service
- .from("credit_usage")
- .select("credits_used")
- .eq("subscription_id", subscription.id)
- .gte("created_at", subscription.current_period_start)
- .lte("created_at", subscription.current_period_end);
-
- if (creditUsageError || !creditUsages) {
- throw new Error(
- `Failed to retrieve credit usage for subscription_id: ${subscription.id}`,
- );
- }
-
- const totalCreditsUsed = creditUsages.reduce(
- (acc, usage) => acc + usage.credits_used,
- 0,
- );
-
- const { data: price, error: priceError } = await supabase_service
- .from("prices")
- .select("credits")
- .eq("id", subscription.price_id)
- .single();
-
- if (priceError || !price) {
- throw new Error(
- `Failed to retrieve price for price_id: ${subscription.price_id}`,
- );
- }
-
- const remainingCredits = price.credits + couponCredits - totalCreditsUsed;
-
- return {
- totalCreditsUsed,
- remainingCredits,
- totalCredits: price.credits,
- };
-}
diff --git a/apps/api/src/services/billing/queue-billing.ts b/apps/api/src/services/billing/queue-billing.ts
deleted file mode 100644
index 5fe109996..000000000
--- a/apps/api/src/services/billing/queue-billing.ts
+++ /dev/null
@@ -1,49 +0,0 @@
-import { logger } from "../../lib/logger";
-import { getBillingQueue } from "../queue-service";
-import { v4 as uuidv4 } from "uuid";
-import * as Sentry from "@sentry/node";
-
-/**
- * Adds a job to the billing queue to trigger batch processing
- * This can be used when we want to ensure billing is processed without waiting for the next interval
- */
-export async function addBillingBatchJob() {
- try {
- const jobId = uuidv4();
- logger.info("Adding billing batch job to queue", { jobId });
-
- await getBillingQueue().add(
- "process-batch",
- {
- timestamp: new Date().toISOString(),
- },
- {
- jobId,
- priority: 10,
- },
- );
-
- return { success: true, jobId };
- } catch (error) {
- logger.error("Error adding billing batch job", { error });
- Sentry.captureException(error, {
- data: {
- operation: "add_billing_batch_job",
- },
- });
- return { success: false, error };
- }
-}
-
-/**
- * Trigger immediate processing of any pending billing operations
- * This is useful for ensuring billing operations are processed without delay
- */
-export async function triggerImmediateBillingProcess() {
- try {
- return await addBillingBatchJob();
- } catch (error) {
- logger.error("Error triggering immediate billing process", { error });
- return { success: false, error };
- }
-}
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index ddaf50392..b41814a4f 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -297,7 +297,7 @@ export async function addIndexInsertJob(data: any) {
);
}
-export async function getIndexInsertJobs(): Promise<any[]> {
+async function getIndexInsertJobs(): Promise<any[]> {
const jobs =
(await redisEvictConnection.lpop(
INDEX_INSERT_QUEUE_KEY,
@@ -345,7 +345,7 @@ export async function addIndexRFInsertJob(data: any) {
);
}
-export async function getIndexRFInsertJobs(): Promise<any[]> {
+async function getIndexRFInsertJobs(): Promise<any[]> {
const jobs =
(await redisEvictConnection.lpop(
INDEX_RF_INSERT_QUEUE_KEY,
@@ -392,7 +392,7 @@ export async function addOMCEJob(data: [number, string]) {
await redisEvictConnection.sadd(OMCE_JOB_QUEUE_KEY, JSON.stringify(data));
}
-export async function getOMCEJobs(): Promise<[number, string][]> {
+async function getOMCEJobs(): Promise<[number, string][]> {
const jobs =
(await redisEvictConnection.spop(
OMCE_JOB_QUEUE_KEY,
@@ -445,7 +445,7 @@ export async function getOMCEQueueLength(): Promise<number> {
const DOMAIN_FREQUENCY_QUEUE_KEY = "domain-frequency-queue";
const DOMAIN_FREQUENCY_BATCH_SIZE = 100;
-export function extractDomainFromUrl(url: string): string | null {
+function extractDomainFromUrl(url: string): string | null {
try {
const urlObj = new URL(url);
// Remove www. prefix for consistency
@@ -469,7 +469,7 @@ export async function addDomainFrequencyJob(url: string) {
await redisEvictConnection.rpush(DOMAIN_FREQUENCY_QUEUE_KEY, domain);
}
-export async function getDomainFrequencyJobs(): Promise<Map<string, number>> {
+async function getDomainFrequencyJobs(): Promise<Map<string, number>> {
const domains =
(await redisEvictConnection.lpop(
DOMAIN_FREQUENCY_QUEUE_KEY,
diff --git a/apps/api/src/services/ledger/data-schemas.ts b/apps/api/src/services/ledger/data-schemas.ts
index dee50ec2b..3eec1a331 100644
--- a/apps/api/src/services/ledger/data-schemas.ts
+++ b/apps/api/src/services/ledger/data-schemas.ts
@@ -6,7 +6,7 @@
/**
* Common properties shared across all events
*/
-export interface BaseEventData {
+/* export */ interface BaseEventData {
user_id?: string; // User ID if available
team_id?: string; // Team ID if available
}
@@ -15,7 +15,8 @@ export interface BaseEventData {
* Concurrent browser limit reached event data
* Tracks when a user reaches the concurrent browser limit
*/
-export interface ConcurrentBrowserLimitReachedEventData extends BaseEventData {
+/* export */ interface ConcurrentBrowserLimitReachedEventData
+ extends BaseEventData {
team_id: string; // The team ID
}
/**
@@ -30,19 +31,19 @@ export interface EventDataMap {
*/
export type EventDefinitionSlug = keyof EventDataMap;
-/**
- * Helper type to extract the data type for a specific event
- */
-export type EventData<T extends EventDefinitionSlug> = EventDataMap[T];
+// /**
+// * Helper type to extract the data type for a specific event
+// */
+// export type EventData<T extends EventDefinitionSlug> = EventDataMap[T];
-/**
- * Creates a properly typed event data object with current timestamp
- */
-export function createEventData<T extends EventDefinitionSlug>(
- eventType: T,
- data: EventDataMap[T],
-): EventDataMap[T] {
- return {
- ...data,
- } as EventDataMap[T];
-}
+// /**
+// * Creates a properly typed event data object with current timestamp
+// */
+// export function createEventData<T extends EventDefinitionSlug>(
+// eventType: T,
+// data: EventDataMap[T],
+// ): EventDataMap[T] {
+// return {
+// ...data,
+// } as EventDataMap[T];
+// }
diff --git a/apps/api/src/services/posthog.ts b/apps/api/src/services/posthog.ts
index 3f56123ca..02c9f7907 100644
--- a/apps/api/src/services/posthog.ts
+++ b/apps/api/src/services/posthog.ts
@@ -2,7 +2,7 @@ import { PostHog } from "posthog-node";
import "dotenv/config";
import { logger } from "../../src/lib/logger";
-export default function PostHogClient(apiKey: string) {
+function PostHogClient(apiKey: string) {
const posthogClient = new PostHog(apiKey, {
host: process.env.POSTHOG_HOST,
flushAt: 1,
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 5e0f68f47..c726ffb54 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -4,8 +4,6 @@ import IORedis from "ioredis";
import { BullMQOtel } from "bullmq-otel";
import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";
-export type QueueFunction = () => Queue<any, any, string, any, any, string>;
-
let extractQueue: Queue;
let loggingQueue: Queue;
let indexQueue: Queue;
@@ -27,12 +25,10 @@ export function getRedisConnection(): IORedis {
return redisConnection;
}
-export const extractQueueName = "{extractQueue}";
-export const loggingQueueName = "{loggingQueue}";
-export const indexQueueName = "{indexQueue}";
-export const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
-export const deepResearchQueueName = "{deepResearchQueue}";
-export const billingQueueName = "{billingQueue}";
+const extractQueueName = "{extractQueue}";
+const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
+const deepResearchQueueName = "{deepResearchQueue}";
+const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
export function getExtractQueue() {
diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts
index b9dfea7b3..2b6bbda05 100644
--- a/apps/api/src/services/rate-limiter.ts
+++ b/apps/api/src/services/rate-limiter.ts
@@ -15,13 +15,6 @@ const createRateLimiter = (keyPrefix, points) =>
duration: 60, // Duration in seconds
});
-export const testSuiteRateLimiter = new RateLimiterRedis({
- storeClient: redisRateLimitClient,
- keyPrefix: "test-suite",
- points: 10000,
- duration: 60, // Duration in seconds
-});
-
const fallbackRateLimits: AuthCreditUsageChunk["rate_limits"] = {
crawl: 15,
scrape: 100,
diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.ts
index 422b91106..4edccaa7d 100644
--- a/apps/api/src/services/webhook/delivery.ts
+++ b/apps/api/src/services/webhook/delivery.ts
@@ -174,7 +174,7 @@ export async function processWebhookInsertJobs() {
}
}
-export async function logWebhook(data: {
+async function logWebhook(data: {
success: boolean;
error?: string;
teamId: string;
diff --git a/apps/api/src/services/webhook/index.ts b/apps/api/src/services/webhook/index.ts
index 6b2ef3bef..f3369ffe8 100644
--- a/apps/api/src/services/webhook/index.ts
+++ b/apps/api/src/services/webhook/index.ts
@@ -29,4 +29,4 @@ export {
getWebhookInsertQueueLength,
processWebhookInsertJobs,
} from "./delivery";
-export * from "./types";
+export { WebhookEvent } from "./types";
diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.ts
index 8582a4b67..92684cce0 100644
--- a/apps/api/src/services/webhook/types.ts
+++ b/apps/api/src/services/webhook/types.ts
@@ -29,13 +29,13 @@ export type WebhookEventDataMap = {
export type WebhookConfig = z.infer<typeof webhookSchema>;
-export interface WebhookDocument {
+interface WebhookDocument {
content?: string;
markdown: string;
metadata: Record<string, any>;
}
-export interface WebhookDocumentLink {
+interface WebhookDocumentLink {
content: WebhookDocument;
source: string;
}
@@ -46,55 +46,50 @@ interface BaseWebhookData {
}
// crawl
-export interface CrawlStartedData extends BaseWebhookData {
+interface CrawlStartedData extends BaseWebhookData {
success: true;
}
-export interface CrawlPageData extends BaseWebhookData {
+interface CrawlPageData extends BaseWebhookData {
success: boolean;
data: Document[] | WebhookDocumentLink[]; // links or documents (v0 compatible)
scrapeId: string;
error?: string;
}
-export interface CrawlCompletedData extends BaseWebhookData {
+interface CrawlCompletedData extends BaseWebhookData {
success: true;
data: Document[] | WebhookDocumentLink[]; // empty array or links (v0 compatible)
}
-export interface CrawlFailedData extends BaseWebhookData {
- success: false;
- error: string;
-}
-
// batch scrape
-export interface BatchScrapeStartedData extends BaseWebhookData {
+interface BatchScrapeStartedData extends BaseWebhookData {
success: true;
}
-export interface BatchScrapePageData extends BaseWebhookData {
+interface BatchScrapePageData extends BaseWebhookData {
success: boolean;
data: Document[];
scrapeId: string;
error?: string; // more v0 tomfoolery
}
-export interface BatchScrapeCompletedData extends BaseWebhookData {
+interface BatchScrapeCompletedData extends BaseWebhookData {
success: true;
data: WebhookDocumentLink[];
}
// extract
-export interface ExtractStartedData extends BaseWebhookData {
+interface ExtractStartedData extends BaseWebhookData {
success: true;
}
-export interface ExtractCompletedData extends BaseWebhookData {
+interface ExtractCompletedData extends BaseWebhookData {
success: true;
data: ExtractResult[];
}
-export interface ExtractFailedData extends BaseWebhookData {
+interface ExtractFailedData extends BaseWebhookData {
success: false;
error: string;
}
diff --git a/apps/api/src/strings.ts b/apps/api/src/strings.ts
deleted file mode 100644
index d5672b82e..000000000
--- a/apps/api/src/strings.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-export const errorNoResults =
- "No results found, please check the URL or contact us at [email protected] to file a ticket.";
-
-export const clientSideError = "client-side exception has occurred";
diff --git a/apps/api/src/supabase_types.ts b/apps/api/src/supabase_types.ts
deleted file mode 100644
index 00b2efbba..000000000
--- a/apps/api/src/supabase_types.ts
+++ /dev/null
@@ -1,1514 +0,0 @@
-export type Json =
- | string
- | number
- | boolean
- | null
- | { [key: string]: Json | undefined }
- | Json[];
-
-export interface Database {
- public: {
- Tables: {
- api_keys: {
- Row: {
- created_at: string | null;
- id: number;
- key: string | null;
- keyType: string;
- name: string | null;
- project_id: number | null;
- };
- Insert: {
- created_at?: string | null;
- id?: number;
- key?: string | null;
- keyType?: string;
- name?: string | null;
- project_id?: number | null;
- };
- Update: {
- created_at?: string | null;
- id?: number;
- key?: string | null;
- keyType?: string;
- name?: string | null;
- project_id?: number | null;
- };
- Relationships: [
- {
- foreignKeyName: "api_keys_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- ];
- };
- company: {
- Row: {
- company_id: number;
- company_name: string | null;
- display_name: string | null;
- is_white_label: boolean | null;
- name: string;
- playground_type: string | null;
- pricing_plan_id: number;
- };
- Insert: {
- company_id?: number;
- company_name?: string | null;
- display_name?: string | null;
- is_white_label?: boolean | null;
- name: string;
- playground_type?: string | null;
- pricing_plan_id?: number;
- };
- Update: {
- company_id?: number;
- company_name?: string | null;
- display_name?: string | null;
- is_white_label?: boolean | null;
- name?: string;
- playground_type?: string | null;
- pricing_plan_id?: number;
- };
- Relationships: [
- {
- foreignKeyName: "company_pricing_plan_id_fkey";
- columns: ["pricing_plan_id"];
- referencedRelation: "pricing_plan";
- referencedColumns: ["id"];
- },
- ];
- };
- constants: {
- Row: {
- created_at: string | null;
- default_prompt: string;
- id: number;
- };
- Insert: {
- created_at?: string | null;
- default_prompt: string;
- id?: number;
- };
- Update: {
- created_at?: string | null;
- default_prompt?: string;
- id?: number;
- };
- Relationships: [];
- };
- conversation: {
- Row: {
- conversation_id: number;
- end_time: string | null;
- experiment_id: string | null;
- project_id: number;
- start_time: string;
- };
- Insert: {
- conversation_id?: number;
- end_time?: string | null;
- experiment_id?: string | null;
- project_id: number;
- start_time: string;
- };
- Update: {
- conversation_id?: number;
- end_time?: string | null;
- experiment_id?: string | null;
- project_id?: number;
- start_time?: string;
- };
- Relationships: [
- {
- foreignKeyName: "conversation_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- ];
- };
- customers: {
- Row: {
- id: string;
- stripe_customer_id: string | null;
- user_id: string | null;
- };
- Insert: {
- id: string;
- stripe_customer_id?: string | null;
- user_id?: string | null;
- };
- Update: {
- id?: string;
- stripe_customer_id?: string | null;
- user_id?: string | null;
- };
- Relationships: [
- {
- foreignKeyName: "customers_id_fkey";
- columns: ["id"];
- referencedRelation: "users";
- referencedColumns: ["id"];
- },
- {
- foreignKeyName: "customers_user_id_fkey";
- columns: ["user_id"];
- referencedRelation: "users";
- referencedColumns: ["id"];
- },
- ];
- };
- data: {
- Row: {
- company_id: number | null;
- content: string | null;
- data_id: string | null;
- date_added: string | null;
- date_modified: string | null;
- embedding: string | null;
- id: number;
- manual_add: boolean | null;
- message_id: number | null;
- project_id: number | null;
- search_index: unknown | null;
- source: string | null;
- source_name: string | null;
- source_rank: number | null;
- source_text: string | null;
- };
- Insert: {
- company_id?: number | null;
- content?: string | null;
- data_id?: string | null;
- date_added?: string | null;
- date_modified?: string | null;
- embedding?: string | null;
- id?: number;
- manual_add?: boolean | null;
- message_id?: number | null;
- project_id?: number | null;
- search_index?: unknown | null;
- source?: string | null;
- source_name?: string | null;
- source_rank?: number | null;
- source_text?: string | null;
- };
- Update: {
- company_id?: number | null;
- content?: string | null;
- data_id?: string | null;
- date_added?: string | null;
- date_modified?: string | null;
- embedding?: string | null;
- id?: number;
- manual_add?: boolean | null;
- message_id?: number | null;
- project_id?: number | null;
- search_index?: unknown | null;
- source?: string | null;
- source_name?: string | null;
- source_rank?: number | null;
- source_text?: string | null;
- };
- Relationships: [
- {
- foreignKeyName: "data_company_id_fkey";
- columns: ["company_id"];
- referencedRelation: "company";
- referencedColumns: ["company_id"];
- },
- {
- foreignKeyName: "data_data_id_fkey";
- columns: ["data_id"];
- referencedRelation: "data_sources";
- referencedColumns: ["id"];
- },
- {
- foreignKeyName: "data_message_id_fkey";
- columns: ["message_id"];
- referencedRelation: "message";
- referencedColumns: ["message_id"];
- },
- {
- foreignKeyName: "data_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- ];
- };
- data_partitioned: {
- Row: {
- company_id: number | null;
- company_name: string;
- content: string | null;
- data_id: string | null;
- date_added: string | null;
- date_modified: string | null;
- embedding: string | null;
- id: number;
- manual_add: boolean | null;
- message_id: number | null;
- project_id: number | null;
- search_index: unknown | null;
- source: string | null;
- source_name: string | null;
- source_rank: number | null;
- source_text: string | null;
- };
- Insert: {
- company_id?: number | null;
- company_name: string;
- content?: string | null;
- data_id?: string | null;
- date_added?: string | null;
- date_modified?: string | null;
- embedding?: string | null;
- id: number;
- manual_add?: boolean | null;
- message_id?: number | null;
- project_id?: number | null;
- search_index?: unknown | null;
- source?: string | null;
- source_name?: string | null;
- source_rank?: number | null;
- source_text?: string | null;
- };
- Update: {
- company_id?: number | null;
- company_name?: string;
- content?: string | null;
- data_id?: string | null;
- date_added?: string | null;
- date_modified?: string | null;
- embedding?: string | null;
- id?: number;
- manual_add?: boolean | null;
- message_id?: number | null;
- project_id?: number | null;
- search_index?: unknown | null;
- source?: string | null;
- source_name?: string | null;
- source_rank?: number | null;
- source_text?: string | null;
- };
- Relationships: [];
- };
- data_sources: {
- Row: {
- created_at: string | null;
- description: string | null;
- id: string;
- name: string | null;
- placeholder: string | null;
- type: string | null;
- };
- Insert: {
- created_at?: string | null;
- description?: string | null;
- id?: string;
- name?: string | null;
- placeholder?: string | null;
- type?: string | null;
- };
- Update: {
- created_at?: string | null;
- description?: string | null;
- id?: string;
- name?: string | null;
- placeholder?: string | null;
- type?: string | null;
- };
- Relationships: [];
- };
- marketing: {
- Row: {
- created_at: string | null;
- id: number;
- message: string | null;
- };
- Insert: {
- created_at?: string | null;
- id?: number;
- message?: string | null;
- };
- Update: {
- created_at?: string | null;
- id?: number;
- message?: string | null;
- };
- Relationships: [];
- };
- mendable_project: {
- Row: {
- company_id: number | null;
- created_at: string | null;
- display_id: string;
- enforce_whitelist: boolean | null;
- id: number;
- isFaqPublic: boolean;
- max_messages_per_month: number | null;
- max_req_per_ip_per_minute: number | null;
- name: string | null;
- prompt_id: number | null;
- support_url: string | null;
- whitelisted_domains: string[] | null;
- };
- Insert: {
- company_id?: number | null;
- created_at?: string | null;
- display_id?: string;
- enforce_whitelist?: boolean | null;
- id?: number;
- isFaqPublic?: boolean;
- max_messages_per_month?: number | null;
- max_req_per_ip_per_minute?: number | null;
- name?: string | null;
- prompt_id?: number | null;
- support_url?: string | null;
- whitelisted_domains?: string[] | null;
- };
- Update: {
- company_id?: number | null;
- created_at?: string | null;
- display_id?: string;
- enforce_whitelist?: boolean | null;
- id?: number;
- isFaqPublic?: boolean;
- max_messages_per_month?: number | null;
- max_req_per_ip_per_minute?: number | null;
- name?: string | null;
- prompt_id?: number | null;
- support_url?: string | null;
- whitelisted_domains?: string[] | null;
- };
- Relationships: [
- {
- foreignKeyName: "mendable_project_company_id_fkey";
- columns: ["company_id"];
- referencedRelation: "company";
- referencedColumns: ["company_id"];
- },
- ];
- };
- message: {
- Row: {
- conversation_id: number;
- embedding: string | null;
- is_taught: boolean | null;
- message: string;
- message_id: number;
- model_configuration: Json | null;
- prompt_text: string | null;
- rating_value: number | null;
- rephrased_text: string | null;
- sender: string;
- timestamp: string;
- };
- Insert: {
- conversation_id: number;
- embedding?: string | null;
- is_taught?: boolean | null;
- message: string;
- message_id?: number;
- model_configuration?: Json | null;
- prompt_text?: string | null;
- rating_value?: number | null;
- rephrased_text?: string | null;
- sender: string;
- timestamp: string;
- };
- Update: {
- conversation_id?: number;
- embedding?: string | null;
- is_taught?: boolean | null;
- message?: string;
- message_id?: number;
- model_configuration?: Json | null;
- prompt_text?: string | null;
- rating_value?: number | null;
- rephrased_text?: string | null;
- sender?: string;
- timestamp?: string;
- };
- Relationships: [
- {
- foreignKeyName: "message_conversation_id_fkey";
- columns: ["conversation_id"];
- referencedRelation: "conversation";
- referencedColumns: ["conversation_id"];
- },
- ];
- };
- model_configuration: {
- Row: {
- created_at: string | null;
- custom_prompt: string | null;
- id: number;
- model_name: string | null;
- project_id: number;
- suggested_questions: string | null;
- support_link: string | null;
- temperature: number;
- };
- Insert: {
- created_at?: string | null;
- custom_prompt?: string | null;
- id?: number;
- model_name?: string | null;
- project_id: number;
- suggested_questions?: string | null;
- support_link?: string | null;
- temperature?: number;
- };
- Update: {
- created_at?: string | null;
- custom_prompt?: string | null;
- id?: number;
- model_name?: string | null;
- project_id?: number;
- suggested_questions?: string | null;
- support_link?: string | null;
- temperature?: number;
- };
- Relationships: [
- {
- foreignKeyName: "model_configuration_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- ];
- };
- monthly_message_counts: {
- Row: {
- message_count: number | null;
- month: number;
- project_id: number;
- year: number;
- };
- Insert: {
- message_count?: number | null;
- month: number;
- project_id: number;
- year: number;
- };
- Update: {
- message_count?: number | null;
- month?: number;
- project_id?: number;
- year?: number;
- };
- Relationships: [
- {
- foreignKeyName: "monthly_message_counts_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- ];
- };
- prices: {
- Row: {
- active: boolean | null;
- currency: string | null;
- description: string | null;
- id: string;
- interval: Database["public"]["Enums"]["pricing_plan_interval"] | null;
- interval_count: number | null;
- metadata: Json | null;
- product_id: string | null;
- trial_period_days: number | null;
- type: Database["public"]["Enums"]["pricing_type"] | null;
- unit_amount: number | null;
- };
- Insert: {
- active?: boolean | null;
- currency?: string | null;
- description?: string | null;
- id: string;
- interval?:
- | Database["public"]["Enums"]["pricing_plan_interval"]
- | null;
- interval_count?: number | null;
- metadata?: Json | null;
- product_id?: string | null;
- trial_period_days?: number | null;
- type?: Database["public"]["Enums"]["pricing_type"] | null;
- unit_amount?: number | null;
- };
- Update: {
- active?: boolean | null;
- currency?: string | null;
- description?: string | null;
- id?: string;
- interval?:
- | Database["public"]["Enums"]["pricing_plan_interval"]
- | null;
- interval_count?: number | null;
- metadata?: Json | null;
- product_id?: string | null;
- trial_period_days?: number | null;
- type?: Database["public"]["Enums"]["pricing_type"] | null;
- unit_amount?: number | null;
- };
- Relationships: [
- {
- foreignKeyName: "prices_product_id_fkey";
- columns: ["product_id"];
- referencedRelation: "products";
- referencedColumns: ["id"];
- },
- ];
- };
- pricing_plan: {
- Row: {
- base_price: number;
- created_at: string | null;
- gpt_35_turbo_price: number | null;
- gpt_4_price: number | null;
- id: number;
- max_messages_per_month: number | null;
- name: string | null;
- price_per_message: number | null;
- stripe_subscription: boolean;
- };
- Insert: {
- base_price?: number;
- created_at?: string | null;
- gpt_35_turbo_price?: number | null;
- gpt_4_price?: number | null;
- id?: number;
- max_messages_per_month?: number | null;
- name?: string | null;
- price_per_message?: number | null;
- stripe_subscription?: boolean;
- };
- Update: {
- base_price?: number;
- created_at?: string | null;
- gpt_35_turbo_price?: number | null;
- gpt_4_price?: number | null;
- id?: number;
- max_messages_per_month?: number | null;
- name?: string | null;
- price_per_message?: number | null;
- stripe_subscription?: boolean;
- };
- Relationships: [];
- };
- products: {
- Row: {
- active: boolean | null;
- description: string | null;
- id: string;
- image: string | null;
- metadata: Json | null;
- name: string | null;
- };
- Insert: {
- active?: boolean | null;
- description?: string | null;
- id: string;
- image?: string | null;
- metadata?: Json | null;
- name?: string | null;
- };
- Update: {
- active?: boolean | null;
- description?: string | null;
- id?: string;
- image?: string | null;
- metadata?: Json | null;
- name?: string | null;
- };
- Relationships: [];
- };
- prompt: {
- Row: {
- name: string | null;
- prompt: string | null;
- prompt_id: number;
- };
- Insert: {
- name?: string | null;
- prompt?: string | null;
- prompt_id?: number;
- };
- Update: {
- name?: string | null;
- prompt?: string | null;
- prompt_id?: number;
- };
- Relationships: [];
- };
- source: {
- Row: {
- content: string | null;
- created_at: string | null;
- id: number;
- link: string | null;
- message_id: number;
- };
- Insert: {
- content?: string | null;
- created_at?: string | null;
- id?: number;
- link?: string | null;
- message_id: number;
- };
- Update: {
- content?: string | null;
- created_at?: string | null;
- id?: number;
- link?: string | null;
- message_id?: number;
- };
- Relationships: [];
- };
- subscriptions: {
- Row: {
- cancel_at: string | null;
- cancel_at_period_end: boolean | null;
- canceled_at: string | null;
- company_id: number | null;
- created: string;
- current_period_end: string;
- current_period_start: string;
- ended_at: string | null;
- id: string;
- metadata: Json | null;
- price_id: string | null;
- quantity: number | null;
- status: Database["public"]["Enums"]["subscription_status"] | null;
- sub_item_id: string | null;
- trial_end: string | null;
- trial_start: string | null;
- usage_pricing_id: string | null;
- user_id: string;
- };
- Insert: {
- cancel_at?: string | null;
- cancel_at_period_end?: boolean | null;
- canceled_at?: string | null;
- company_id?: number | null;
- created?: string;
- current_period_end?: string;
- current_period_start?: string;
- ended_at?: string | null;
- id: string;
- metadata?: Json | null;
- price_id?: string | null;
- quantity?: number | null;
- status?: Database["public"]["Enums"]["subscription_status"] | null;
- sub_item_id?: string | null;
- trial_end?: string | null;
- trial_start?: string | null;
- usage_pricing_id?: string | null;
- user_id: string;
- };
- Update: {
- cancel_at?: string | null;
- cancel_at_period_end?: boolean | null;
- canceled_at?: string | null;
- company_id?: number | null;
- created?: string;
- current_period_end?: string;
- current_period_start?: string;
- ended_at?: string | null;
- id?: string;
- metadata?: Json | null;
- price_id?: string | null;
- quantity?: number | null;
- status?: Database["public"]["Enums"]["subscription_status"] | null;
- sub_item_id?: string | null;
- trial_end?: string | null;
- trial_start?: string | null;
- usage_pricing_id?: string | null;
- user_id?: string;
- };
- Relationships: [
- {
- foreignKeyName: "subscriptions_company_id_fkey";
- columns: ["company_id"];
- referencedRelation: "company";
- referencedColumns: ["company_id"];
- },
- {
- foreignKeyName: "subscriptions_price_id_fkey";
- columns: ["price_id"];
- referencedRelation: "prices";
- referencedColumns: ["id"];
- },
- {
- foreignKeyName: "subscriptions_user_id_fkey";
- columns: ["user_id"];
- referencedRelation: "users";
- referencedColumns: ["id"];
- },
- ];
- };
- suggested_questions: {
- Row: {
- created_at: string | null;
- id: number;
- project_id: number | null;
- question: string | null;
- };
- Insert: {
- created_at?: string | null;
- id?: number;
- project_id?: number | null;
- question?: string | null;
- };
- Update: {
- created_at?: string | null;
- id?: number;
- project_id?: number | null;
- question?: string | null;
- };
- Relationships: [
- {
- foreignKeyName: "suggested_questions_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- ];
- };
- user_notifications: {
- Row: {
- company_id: number;
- id: number;
- notification_type: string;
- project_id: number;
- sent_date: string;
- user_id: string;
- };
- Insert: {
- company_id: number;
- id?: number;
- notification_type: string;
- project_id: number;
- sent_date: string;
- user_id: string;
- };
- Update: {
- company_id?: number;
- id?: number;
- notification_type?: string;
- project_id?: number;
- sent_date?: string;
- user_id?: string;
- };
- Relationships: [
- {
- foreignKeyName: "user_notifications_company_id_fkey";
- columns: ["company_id"];
- referencedRelation: "company";
- referencedColumns: ["company_id"];
- },
- {
- foreignKeyName: "user_notifications_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- {
- foreignKeyName: "user_notifications_user_id_fkey";
- columns: ["user_id"];
- referencedRelation: "users";
- referencedColumns: ["id"];
- },
- ];
- };
- users: {
- Row: {
- avatar_url: string | null;
- billing_address: Json | null;
- company_id: number | null;
- email: string | null;
- full_name: string | null;
- id: string;
- payment_method: Json | null;
- };
- Insert: {
- avatar_url?: string | null;
- billing_address?: Json | null;
- company_id?: number | null;
- email?: string | null;
- full_name?: string | null;
- id: string;
- payment_method?: Json | null;
- };
- Update: {
- avatar_url?: string | null;
- billing_address?: Json | null;
- company_id?: number | null;
- email?: string | null;
- full_name?: string | null;
- id?: string;
- payment_method?: Json | null;
- };
- Relationships: [
- {
- foreignKeyName: "users_company_id_fkey";
- columns: ["company_id"];
- referencedRelation: "company";
- referencedColumns: ["company_id"];
- },
- {
- foreignKeyName: "users_id_fkey";
- columns: ["id"];
- referencedRelation: "users";
- referencedColumns: ["id"];
- },
- ];
- };
- z_testcomp_92511: {
- Row: {
- content: string | null;
- data_id: string | null;
- date_added: string | null;
- date_modified: string | null;
- embedding: string | null;
- id: number;
- manual_add: boolean | null;
- message_id: number | null;
- project_id: number | null;
- search_index: unknown | null;
- source: string | null;
- source_name: string | null;
- source_rank: number | null;
- source_text: string | null;
- };
- Insert: {
- content?: string | null;
- data_id?: string | null;
- date_added?: string | null;
- date_modified?: string | null;
- embedding?: string | null;
- id?: number;
- manual_add?: boolean | null;
- message_id?: number | null;
- project_id?: number | null;
- search_index?: unknown | null;
- source?: string | null;
- source_name?: string | null;
- source_rank?: number | null;
- source_text?: string | null;
- };
- Update: {
- content?: string | null;
- data_id?: string | null;
- date_added?: string | null;
- date_modified?: string | null;
- embedding?: string | null;
- id?: number;
- manual_add?: boolean | null;
- message_id?: number | null;
- project_id?: number | null;
- search_index?: unknown | null;
- source?: string | null;
- source_name?: string | null;
- source_rank?: number | null;
- source_text?: string | null;
- };
- Relationships: [
- {
- foreignKeyName: "z_testcomp_92511_data_id_fkey";
- columns: ["data_id"];
- referencedRelation: "data_sources";
- referencedColumns: ["id"];
- },
- {
- foreignKeyName: "z_testcomp_92511_message_id_fkey";
- columns: ["message_id"];
- referencedRelation: "message";
- referencedColumns: ["message_id"];
- },
- {
- foreignKeyName: "z_testcomp_92511_project_id_fkey";
- columns: ["project_id"];
- referencedRelation: "mendable_project";
- referencedColumns: ["id"];
- },
- ];
- };
- };
- Views: {
- [_ in never]: never;
- };
- Functions: {
- combine_search_29: {
- Args: {
- query_embedding: string;
- query_text: string;
- table_name: string;
- project_id: number;
- similarity_threshold: number;
- search_thres: number;
- k?: number;
- };
- Returns: {
- id: number;
- content: string;
- source: string;
- similarity: number;
- rank: number;
- similarity_rank: number;
- rank_rank: number;
- rrf_score: number;
- }[];
- };
- create_company_table_2: {
- Args: {
- company: string;
- };
- Returns: undefined;
- };
- create_indexs: {
- Args: {
- tbl_name: string;
- };
- Returns: undefined;
- };
- create_mendable_project_3: {
- Args: {
- _company_name: string;
- _pricing_plan_id: number;
- _project_name: string;
- };
- Returns: Json;
- };
- create_new_company_table_5: {
- Args: {
- _company_name: string;
- _company_display_name: string;
- };
- Returns: number;
- };
- create_new_project_2: {
- Args: {
- _company_id: number;
- _project_name: string;
- };
- Returns: Json;
- };
- create_project_with_api_key_10: {
- Args: {
- _company_name: string;
- _pricing_plan_id: number;
- _project_name: string;
- };
- Returns: Json;
- };
- create_project_with_api_key_8: {
- Args: {
- _company_name: string;
- _pricing_plan_id: number;
- _project_name: string;
- };
- Returns: boolean;
- };
- create_table: {
- Args: {
- table_name: string;
- };
- Returns: undefined;
- };
- dmetaphone: {
- Args: {
- "": string;
- };
- Returns: string;
- };
- dmetaphone_alt: {
- Args: {
- "": string;
- };
- Returns: string;
- };
- get_all_conversation_with_messages_by_project_id: {
- Args: {
- _project_id: number;
- page: number;
- conversations_per_page: number;
- };
- Returns: {
- conversation_id: number;
- end_time: string | null;
- experiment_id: string | null;
- project_id: number;
- start_time: string;
- }[];
- };
- get_index_types: {
- Args: {
- _company_name: string;
- _project_id: number;
- };
- Returns: unknown;
- };
- get_message_count_by_project: {
- Args: {
- p_project_id: number;
- };
- Returns: {
- message_day: string;
- message_count: number;
- }[];
- };
- get_messages_by_project_and_rating: {
- Args: {
- _project_id: number;
- _rating_value: number;
- current_page: number;
- pages_per_set: number;
- };
- Returns: {
- message_id: number;
- conversation_id: number;
- message: string;
- timestamp: string;
- sender: string;
- rating_value: number;
- prev_message_id: number;
- prev_message: string;
- prev_timestamp: string;
- prev_sender: string;
- }[];
- };
- get_messages_without_sources_by_project_with_prev: {
- Args: {
- _project_id: number;
- current_page: number;
- pages_per_set: number;
- };
- Returns: {
- message_id: number;
- conversation_id: number;
- message: string;
- timestamp: string;
- sender: string;
- rating_value: number;
- prev_message_id: number;
- prev_message: string;
- prev_timestamp: string;
- prev_sender: string;
- }[];
- };
- get_messages_without_sources_count_by_project: {
- Args: {
- _project_id: number;
- };
- Returns: number;
- };
- get_negative_messages_count_by_project_id: {
- Args: {
- _project_id: number;
- };
- Returns: number;
- };
- get_positive_messages_count_by_project: {
- Args: {
- _project_id: number;
- };
- Returns: number;
- };
- get_project_and_company_with_model_configuration: {
- Args: {
- p_project_id: number;
- };
- Returns: {
- project: unknown;
- company: unknown;
- model_configuration: unknown;
- }[];
- };
- get_project_company_model: {
- Args: {
- api_key_in: string;
- };
- Returns: {
- company: unknown;
- project: unknown;
- configuration: unknown;
- }[];
- };
- get_project_message_count: {
- Args: {
- p_project_id: number;
- p_start_date: string;
- p_end_date: string;
- };
- Returns: number;
- };
- get_sources_count: {
- Args: {
- company_name: string;
- project_id: number;
- };
- Returns: number;
- };
- get_sources_from_message: {
- Args: {
- mid: number;
- };
- Returns: {
- content: string | null;
- created_at: string | null;
- id: number;
- link: string | null;
- message_id: number;
- }[];
- };
- get_sources_info_new: {
- Args: {
- company_name: string;
- project_id: number;
- pages_per_set: number;
- current_page: number;
- };
- Returns: {
- source: string;
- source_rank: number;
- data_id: string;
- source_text: string;
- source_name: string;
- }[];
- };
- get_subscription: {
- Args: {
- p_project_id: number;
- };
- Returns: {
- cancel_at: string | null;
- cancel_at_period_end: boolean | null;
- canceled_at: string | null;
- company_id: number | null;
- created: string;
- current_period_end: string;
- current_period_start: string;
- ended_at: string | null;
- id: string;
- metadata: Json | null;
- price_id: string | null;
- quantity: number | null;
- status: Database["public"]["Enums"]["subscription_status"] | null;
- sub_item_id: string | null;
- trial_end: string | null;
- trial_start: string | null;
- usage_pricing_id: string | null;
- user_id: string;
- };
- };
- get_total_conversations_count_by_project: {
- Args: {
- _project_id: number;
- };
- Returns: number;
- };
- get_trained_messages_from_project: {
- Args: {
- p_project_id: number;
- current_page: number;
- pages_per_set: number;
- };
- Returns: {
- current_message: Json;
- preceding_message: Json;
- }[];
- };
- get_trained_messages_from_project_count: {
- Args: {
- p_project_id: number;
- };
- Returns: number;
- };
- getPricingPlanByName: {
- Args: {
- company_name: string;
- };
- Returns: {
- base_price: number;
- created_at: string | null;
- gpt_35_turbo_price: number | null;
- gpt_4_price: number | null;
- id: number;
- max_messages_per_month: number | null;
- name: string | null;
- price_per_message: number | null;
- stripe_subscription: boolean;
- };
- };
- gtrgm_compress: {
- Args: {
- "": unknown;
- };
- Returns: unknown;
- };
- gtrgm_decompress: {
- Args: {
- "": unknown;
- };
- Returns: unknown;
- };
- gtrgm_in: {
- Args: {
- "": unknown;
- };
- Returns: unknown;
- };
- gtrgm_options: {
- Args: {
- "": unknown;
- };
- Returns: undefined;
- };
- gtrgm_out: {
- Args: {
- "": unknown;
- };
- Returns: unknown;
- };
- increment_message_count: {
- Args: {
- projectid: number;
- month: number;
- year: number;
- };
- Returns: undefined;
- };
- increment_message_count_2: {
- Args: {
- p_projectid: number;
- p_month: number;
- p_year: number;
- };
- Returns: undefined;
- };
- ivfflathandler: {
- Args: {
- "": unknown;
- };
- Returns: unknown;
- };
- keyword_search_trigram22: {
- Args: {
- k: number;
- query_text: string;
- table_name: string;
- project_id: number;
- limit_results: number;
- };
- Returns: {
- id: number;
- content: string;
- source: string;
- similarity_score: number;
- }[];
- };
- match_documents: {
- Args: {
- table_name: string;
- query_embedding: string;
- similarity_threshold: number;
- match_count: number;
- };
- Returns: {
- id: number;
- content: string;
- similarity: number;
- source: string;
- }[];
- };
- messagesSentThisMonthByName: {
- Args: {
- company_name: string;
- };
- Returns: number;
- };
- search_content_priority_by_project_2: {
- Args: {
- tbl_name: string;
- query_text: string;
- search_thres: number;
- num_results: number;
- project_id: number;
- };
- Returns: {
- id: number;
- content: string;
- source: string;
- rank: number;
- }[];
- };
- search_content_priority30: {
- Args: {
- query_text: string;
- tbl_name: string;
- num_results: number;
- search_thres: number;
- };
- Returns: {
- id: number;
- content: string;
- source: string;
- rank: number;
- }[];
- };
- search_documents: {
- Args: {
- k: number;
- query_embedding: string;
- query_text: string;
- table_name: string;
- project_id: number;
- limit_results: number;
- };
- Returns: {
- id: number;
- content: string;
- source: string;
- similarity: number;
- rank: number;
- similarity_rank: number;
- rank_rank: number;
- rrf_score: number;
- }[];
- };
- search_documents_return_all5: {
- Args: {
- k: number;
- query_embedding: string;
- query_text: string;
- table_name: string;
- project_id: number;
- limit_results: number;
- };
- Returns: {
- id: number;
- content: string;
- source: string;
- similarity: number;
- rank: number;
- similarity_rank: number;
- rank_rank: number;
- rrf_score: number;
- source_name: string;
- date_added: string;
- date_modified: string;
- }[];
- };
- set_limit: {
- Args: {
- "": number;
- };
- Returns: number;
- };
- show_limit: {
- Args: Record<PropertyKey, never>;
- Returns: number;
- };
- show_trgm: {
- Args: {
- "": string;
- };
- Returns: unknown;
- };
- soundex: {
- Args: {
- "": string;
- };
- Returns: string;
- };
- text_soundex: {
- Args: {
- "": string;
- };
- Returns: string;
- };
- update_search_index: {
- Args: {
- table_name: string;
- };
- Returns: undefined;
- };
- update_search_index_2: {
- Args: {
- table_name: string;
- };
- Returns: Json;
- };
- update_search_index_single_row: {
- Args: {
- table_name: string;
- row_id: number;
- };
- Returns: Json;
- };
- vector_avg: {
- Args: {
- "": number[];
- };
- Returns: string;
- };
- vector_dims: {
- Args: {
- "": string;
- };
- Returns: number;
- };
- vector_norm: {
- Args: {
- "": string;
- };
- Returns: number;
- };
- vector_out: {
- Args: {
- "": string;
- };
- Returns: unknown;
- };
- vector_send: {
- Args: {
- "": string;
- };
- Returns: string;
- };
- vector_typmod_in: {
- Args: {
- "": unknown[];
- };
- Returns: number;
- };
- get_documents: {
- Args: {
- c_company_id;
- page_size;
- z_offset;
- };
- };
- };
- Enums: {
- pricing_plan_interval: "day" | "week" | "month" | "year";
- pricing_type: "one_time" | "recurring";
- subscription_status:
- | "trialing"
- | "active"
- | "canceled"
- | "incomplete"
- | "incomplete_expired"
- | "past_due"
- | "unpaid"
- | "paused";
- };
- CompositeTypes: {
- [_ in never]: never;
- };
- };
-}
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index b5f99a7e9..28aced098 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -15,24 +15,6 @@ type Mode = "crawl" | "single_urls" | "sitemap" | "kickoff";
export { Mode };
-export interface CrawlResult {
- source: string;
- content: string;
- options?: {
- summarize?: boolean;
- summarize_max_chars?: number;
- };
- metadata?: any;
- raw_context_id?: number | string;
- permissions?: any[];
-}
-
-export interface IngestResult {
- success: boolean;
- error: string;
- data: CrawlResult[];
-}
-
export interface WebScraperOptions {
url: string;
mode: Mode;
@@ -77,16 +59,6 @@ export interface RunWebScraperParams {
costTracking: CostTracking;
}
-export type RunWebScraperResult =
- | {
- success: false;
- error: Error;
- }
- | {
- success: true;
- document: V2Document;
- };
-
export interface FirecrawlJob {
job_id?: string;
success: boolean;
@@ -142,15 +114,6 @@ export interface FirecrawlCrawlStatusResponse {
error?: string;
}
-export interface FirecrawlExtractResponse {
- statusCode: number;
- body: {
- success: boolean;
- data: any[];
- };
- error?: string;
-}
-
export enum RateLimiterMode {
Crawl = "crawl",
CrawlStatus = "crawlStatus",
@@ -186,18 +149,3 @@ export enum NotificationType {
CONCURRENCY_LIMIT_REACHED = "concurrencyLimitReached",
AUTO_RECHARGE_FREQUENT = "autoRechargeFrequent",
}
-
-export type ScrapeLog = {
- url: string;
- scraper: string;
- success?: boolean;
- response_code?: number;
- time_taken_seconds?: number;
- proxy?: string;
- retried?: boolean;
- error_message?: string;
- date_added?: string; // ISO 8601 format
- html?: string;
- ipv4_support?: boolean | null;
- ipv6_support?: boolean | null;
-};
diff --git a/apps/api/src/utils/integration.ts b/apps/api/src/utils/integration.ts
index 8263a0a81..d44b66520 100644
--- a/apps/api/src/utils/integration.ts
+++ b/apps/api/src/utils/integration.ts
@@ -1,6 +1,6 @@
import { z } from "zod";
-export enum IntegrationEnum {
+enum IntegrationEnum {
DIFY = "dify",
ZAPIER = "zapier",
PIPEDREAM = "pipedream",