Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: wip

Nicolas78badf8f7265
diff --git a/apps/api/package.json b/apps/api/package.json
index a0f9cf8e5..dc62d4c35 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -53,7 +53,7 @@
"@bull-board/api": "^5.20.5",
"@bull-board/express": "^5.20.5",
"@devil7softwares/pos": "^1.0.2",
- "@dqbd/tiktoken": "^1.0.13",
+ "@dqbd/tiktoken": "^1.0.17",
"@hyperdx/node-opentelemetry": "^0.8.1",
"@logtail/node": "^0.4.12",
"@nangohq/node": "^0.40.8",
@@ -73,6 +73,7 @@
"cacheable-lookup": "^6.1.0",
"cheerio": "^1.0.0-rc.12",
"cohere": "^1.1.1",
+ "cohere-ai": "^7.14.0",
"cors": "^2.8.5",
"cron-parser": "^4.9.0",
"date-fns": "^3.6.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 095b507c3..f07c4ecda 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -24,8 +24,8 @@ importers:
specifier: ^1.0.2
version: 1.0.2
'@dqbd/tiktoken':
- specifier: ^1.0.13
- version: 1.0.15
+ specifier: ^1.0.17
+ version: 1.0.17
'@hyperdx/node-opentelemetry':
specifier: ^0.8.1
version: 0.8.1
@@ -83,6 +83,9 @@ importers:
cohere:
specifier: ^1.1.1
version: 1.1.1
+ cohere-ai:
+ specifier: ^7.14.0
+ version: 7.14.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
cors:
specifier: ^2.8.5
version: 2.8.5
@@ -130,7 +133,7 @@ importers:
version: 2.9.0
langchain:
specifier: ^0.2.8
languagedetect:
specifier: ^2.0.0
version: 2.0.0
@@ -148,10 +151,10 @@ importers:
version: 2.30.1
mongoose:
specifier: ^8.4.4
- version: 8.4.4([email protected])
+ version: 8.4.4(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])
natural:
specifier: ^7.0.7
- version: 7.0.7([email protected])
+ version: 7.0.7(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])
openai:
specifier: ^4.57.0
version: 4.57.0([email protected])
@@ -289,6 +292,154 @@ packages:
'@anthropic-ai/[email protected]':
resolution: {integrity: sha512-916wJXO6T6k8R6BAAcLhLPv/pnLGy7YSEBZXZ1XTFbLcTZE8oTy3oDW9WJf9KKZwMvVcePIfoTSvzXHRcGxkQQ==}
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-vJzQ6QpaMu8itJMe3FH1/0rwMjL0ELh63iLTxiAmhiV/SvCwNNoSFLd2HdKxbV0Bg/x8lUiPVq3pl6+cxaIrEQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-n1hTHpQl6LwNkwn4vLmtbwkNoX2jxtiliRd0IaHR1CfAQvKNTfQ52mARWr73hR+/YcVsBzPx8sYKq2XHWArHKQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-/dBYWcCwbA/id4sFCIVZvf0UsvzHCC68SryxeNQk/PDkY9N4n5yRcMUkZDaEyQCjowc3kY4JOXp2AdUP037nhA==}
+ engines: {node: '>=16.0.0'}
+ peerDependencies:
+ '@aws-sdk/client-sts': ^3.679.0
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-/0cAvYnpOZTo/Y961F1kx2fhDDLUYZ0SQQ5/75gh3xVImLj7Zw+vp74ieqFbqWLYGMaq8z1Arr9A8zG95mbLdg==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-3CvrT8w1RjFu1g8vKA5Azfr5V83r2/b68Ock43WE003Bq/5Y38mwmYX7vk0fPHzC3qejt4YMAWk/C3fSKOy25g==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-CS6PWGX8l4v/xyvX8RtXnBisdCa5+URzKd0L6GvHChype9qKUVxO/Gg6N/y43Hvg7MNWJt9FBPNWIxUB+byJwg==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-XvWd6RPk7TA7tmqITT+NXvJ6ltJP8BUtLO1NAvja4HKExPKR9HAyoOeeH7KM3lVRED4e4LUnLb3fzteH20IXaA==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-EdlTYbzMm3G7VUNAMxr9S1nC1qUNqhKlAxFU8E7cKsAe8Bp29CD5HAs3POc56AVo9GC4yRIS+/mtlZSmrckzUA==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-ZoKLubW5DqqV1/2a3TSn+9sSKg0T8SsYMt1JeirnuLJF0mCoYFUaWMyvxxKuxPoqvUsaycxKru4GkpJ10ltNBw==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-Rg7t8RwUzKcumpipG4neZqaeJ6DF+Bco1+FHn5BZB68jpvwvjBjcQUuWkxj18B6ctYHr1fkunnzeKEn/+vy7+w==}
+ engines: {node: '>=16.0.0'}
+ peerDependencies:
+ '@aws-sdk/client-sts': ^3.679.0
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-E3lBtaqCte8tWs6Rkssc8sLzvGoJ10TLGvpkijOlz43wPd6xCRh1YLwg6zolf9fVFtEyUs/GsgymiASOyxhFtw==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-u/p4TV8kQ0zJWDdZD4+vdQFTMhkDEJFws040Gm113VHa/Xo1SYOjbpvqeuFoz6VmM0bLvoOWjxB9MxnSQbwKpQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-SAtWonhi9asxn0ukEbcE81jkyanKgqpsrtskvYPpO9Z9KOednM4Cqt6h1bfcS9zaHjN2zu815Gv8O7WiV+F/DQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-a74tLccVznXCaBefWPSysUcLXYJiSkeUmQGtalNgJ1vGkE36W5l/8czFiiowdWdKWz7+x6xf0w+Kjkjlj42Ung==}
+ engines: {node: '>=16.0.0'}
+ peerDependencies:
+ '@aws-sdk/client-sts': ^3.679.0
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-ZjZZb6OERw/UKlSqcJ24AUJIf/ekDLPZrPpo0kPMV70EQ0GkBiklIZ8qULu9bEcI2I4UIapBKRiXTrK4gA6YHg==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-y176HuQ8JRY3hGX8rQzHDSbCl9P5Ny9l16z4xmaiLo+Qfte7ee4Yr3yaAKd7GFoJ3/Mhud2XZ37fR015MfYl2w==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-0vet8InEj7nvIvGKk+ch7bEF5SyZ7Us9U7YTEgXPrBNStKeRUsgwRm0ijPWWd0a3oz2okaEwXsFl7G/vI0XiEA==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-sQoAZFsQiW/LL3DfKMYwBoGjYDEnMbA9WslWN8xneCmBAwKo6IcSksvYs23PP8XMIoBGe2I2J9BSr654XWygTQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-4hdeXhPDURPqQLPd9jCpUEo9fQITXl3NM3W1MwcJpE0gdUM36uXkQOYsTPeeU/IRCLVjK8Htlh2oCaM9iJrLCA==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-9WpRUbINdGroV3HiZZIBoJvL2ndoWk39OfwxWs2otxByppJZNN14bg/lvCx5e8ggHUti7IBk5rb0nqQZ4m05pg==}
+ engines: {node: '>=14.0.0'}
+ deprecated: This package has moved to @smithy/protocol-http
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-Ybx54P8Tg6KKq5ck7uwdjiKif7n/8g1x+V0V9uTjBjRWqaIgiqzXwKWoPj6NCNkE7tJNtqI4JrNxp/3S3HvmRw==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-2xLJvSdzcZZAg0lsDLUAuSQuihzK0dcxIK7WmfuJeF7DGKJFmp9czQmz5f3qiDz6IDQzvgK1M9vtJSVCslJbyQ==}
+ engines: {node: '>=14.0.0'}
+ deprecated: This package has moved to @smithy/signature-v4
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-1/+Zso/x2jqgutKixYFQEGli0FELTgah6bm7aB+m2FAWH4Hz7+iMUsazg6nSWm714sG9G3h5u42Dmpvi9X6/hA==}
+ engines: {node: '>=16.0.0'}
+ peerDependencies:
+ '@aws-sdk/client-sso-oidc': ^3.679.0
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-NwVq8YvInxQdJ47+zz4fH3BRRLC6lL+WLkvr242PVBbUOLRyK/lkwHlfiKUoeVIMyK5NF+up6TRg71t/8Bny6Q==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-YL6s4Y/1zC45OvddvgE139fjeWSKKPgLlnfrvhVL7alNyY9n7beR4uhoDpNrt5mI6sn9qiBF17790o+xLAXjjg==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-zKTd48/ZWrCplkXpYDABI74rQlbR0DNHs8nH95htfSLj9/mWRSwaGptoxwcihaq/77vi/fl2X3y0a1Bo8bt7RA==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-CusSm2bTBG1kFypcsqU8COhnYc6zltobsqs3nRrvYqYaOqtMnuE46K4XTWpnzKgwDejgZGOE+WYyprtAxrPvmQ==}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-Bw4uXZ+NU5ed6TNfo4tBbhBSW+2eQxXYjYBGl5gLUNUpg2pDFToQAP6rXBFiwcG52V2ny5oLGiD82SoYuYkAVg==}
+ engines: {node: '>=16.0.0'}
+ peerDependencies:
+ aws-crt: '>=1.0.0'
+ peerDependenciesMeta:
+ aws-crt:
+ optional: true
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==}
+
resolution: {integrity: sha512-ZJhac6FkEd1yhG2AHOmfcXG4ceoLltoCVJjN5XsWN9BifBQr+cHJbWi0h68HZuSORq+3WtJ2z0hwF2NG1b5kcA==}
engines: {node: '>=6.9.0'}
@@ -499,8 +650,8 @@ packages:
engines: {node: '>=0'}
deprecated: This package has been renamed to `fast-tag-pos`
- resolution: {integrity: sha512-a6I67K1xUkuqcuwulobIJiLikkoE7egMaviI1Jg5bxSn2V7QGqXsGE3jTKr8UIOU/o74mAAd5TkeXFNBtaKF4A==}
+ resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==}
'@flydotio/[email protected]':
resolution: {integrity: sha512-L52UAfrOhmAn3T4TxpeRofQOSO+Kctg+uraB4nLzo4mvvh+4Z7HYxSi7Dnq0Kirz+xx6fDIc4OMNT1EdaORecA==}
@@ -1475,6 +1626,216 @@ packages:
'@sinonjs/[email protected]':
resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==}
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-0XuhuHQlEqbNQZp7QxxrFTdVWdwxch4vjxYgfInF91hZFkPxf9QDrdQka0KfxFMPqLNzSw0b95uGTrLliQUavQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-Uh0Sz9gdUuz538nvkPiyv1DZRX9+D15EKDtnQP5rYVAzM/dnYk3P8cg73jcxyOitPgT3mE3OVj7ky7sibzHWkw==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-DujtuDA7BGEKExJ05W5OdxCoyekcKT3Rhg1ZGeiUWaz2BJIWXjZmsG/DIP4W48GHno7AQwRsaCb8NcBgH3QZpg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-4FTQGAsuwqTzVMmiRVTn0RR9GrbRfkP0wfu/tXWVHd2LgNpTY0uglQpIScXK4NaEyXbB3JmZt8gfVqO50lP8wg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-3tEbUb8t8an226jKB6V/Q2XU/J53lCwCzULuBPEaF4JjSh+FlCMp7TmogE/Aij5J9DwlsZ4VAD/IRDuQ/0ZtMw==}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-hYNVQOqhFQ6vOpenifFME546f0GfJn2OiQ3M0FDmuUu8V/Uiwy2wej7ZXxFBNqdx0R5DZAqWM1l6VRhGz8oE6A==}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-MLb1f5tbBO2X6K4lMEKJvxeLooyg7guq48C2zKr4qM7F2Gpkz4dc+hdSgu77pCJ76jVqFBjZczHYAs6dp15N+g==}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-tlNQYbfpWXHimHqrvgo14DrMAgUBua/cNoz9fMYcDmYej7MAmUcjav/QKQbFc3NrcPxeJ7QClER4tWZmfwoPng==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-7Qynk6NWtTQhnGTTZwks++nJhQ1O54Mzi7fz4PqZOiYXb4Z1Flpb2yRvdALoggTS8xjtohWUM+RygOtB30YL3Q==}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-twpQ/n+3OWZJ7Z+xu43MJErmhB/WO/mMTnqR6PwWQShvSJ/emx5d1N59LQZk6ZpTAeuRWrc+eHhkzTp9NFjNRQ==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-T4dIdCs1d/+/qMpwhJ1DzOhxCZjZHbHazEPJWdB4GDi2HjIZllVzeBEcdJUN0fomV8DURsgOyrbEUzg3vzTaOg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-wWO3xYmFm6WRW8VsEJ5oU6h7aosFXfszlz3Dj176pTij6o21oZnzkCLzShfmRaaCHDkBXWBdO0c4sQAvLFP6zA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-m1F70cPaMBML4HiTgCw5I+jFNtjgz5z5UdGnUbG37vw6kh4UvizFYjqJGHvicfgKMkDL6mXwyPp5mhZg02g5sg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-Xg2jK9Wc/1g/MBMP/EUn2DLspN8LNt+GMe7cgF+Ty3vl+Zvu+VeZU5nmhveU+H8pxyTsjrAkci8NqY6OuvZnjA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-d7ZuwvYgp1+3682Nx0MD3D/HtkmZd49N3JUndYWQXfRZrYEnCWYc8BHcNmVsPAp9gKvlurdg/mubE6b/rPS9MA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-qRHoah49QJ71eemjuS/WhUXB+mpNtwHRWQr77J/m40ewBVVwvo52kYAmb7iuaECgGTTcYxHS4Wmewfwy++ueew==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-PkOwPNeKdvX/jCpn0A8n9/TyoxjGZB8WVoJmm9YzsnAgggTj4CrjpRHlTQw7dlLZ320n1mY1y+nTRUDViKi/3w==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-ukNUyo6rHmusG64lmkjFeXemwYuKge1BJ8CtpVKmrxQxc6rhUX0vebcptFA9MmrGsnLhwnnqeH83VTU9hwOpjA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-GfGfruksi3nXdFok5RhgtOnWe5f6BndzYfmEXISD+5gAGdayFGpjWu5pIqIweTudMtse20bGbc+7MFZXT1Tb8Q==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-hsjtwpIemmCkm3ZV5fd/T0bPIugW1gJXwZ/hpuVubt2hEUApIoUTrf6qIdh9MAWlw0vjMrA1ztJLAwtNaZogvg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-btYxGVqFUARbUrN6VhL9c3dnSviIwBYD9Rz1jHuN1hgh28Fpv2xjU1HeCeDJX68xctz7r4l1PBnFhGg1WBBPuA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-BtEk3FG7Ks64GAbt+JnKqwuobJNX8VmFLBsKIwWr1D60T426fGrV2L3YS5siOcUhhp6/Y6yhBw1PSPxA5p7qGg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-uEC/kCCFto83bz5ZzapcrgGqHOh/0r69sZ2ZuHlgoD5kYgXJEThCoTuw/y1Ub3cE7aaKdznb+jD9xRPIfIwD7g==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-/+OsJRNtoRbtsX0UpSgWVxFZLsJHo/4sTr+kBg/J78sr7iC+tHeOvOJrS5hCpVQ6sWBbhWLp1UNiuMyZhE6pmA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-fDo3m7YqXBs7neciOePPd/X9LPm5QLlDMdIC4m1H6dgNLnXfLMFNIxEfPyohGA8VW9Wn4X8lygnPSGxDZSmp0Q==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-NsV1jF4EvmO5wqmaSzlnTVetemBS3FZHdyc5CExbDljcyJCEEkJr8ANu2JvtNbVg/9MvKAWV44kTrGS+Pi4INg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-dxw1BDxJiY9/zI3cBqfVrInij6ShjpV4fmGHesGZZUiP9OSE/EVfdwdRz0PgvkEvrZHpsj2htRaHJfftE8giBA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-8VXK/KzOHefoC65yRgCn5vG1cysPJjHnOVt9d0ybFQSmJgQj152vMn4EkYhGuaOmnnZvCPav/KnYyE6/KsNZ2w==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-4FdOhwpTW7jtSFWm7SpfLGKIBC9ZaTKG5nBF0wK24aoQKQyDIKUw3+KFWCQ9maMzrgTJIuOvOnsV2lLGW5XjTg==}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-9m6NXE0ww+ra5HKHCHig20T+FAwxBAm7DIdwc/767uGWbRcY720ybgPacQNB96JMOI7xVr/CDa3oMzKmW4a+kw==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-fRw7zymjIDt6XxIsLwfJfYUfbGoO9CmCJk6rjJ/X5cd20+d2Is7xjU5Kt/AiDt6hX8DAf5dztmfP5O82gR9emA==}
+ engines: {node: '>= 10.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-H3BSZdBDiVZGzt8TG51Pd2FvFO0PAx/A0mJ0EH8a13KJ6iUCdYnw/Dk/MdC1kTd0eUuUGisDFaxXVXo4HHFL1g==}
+ engines: {node: '>= 10.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-kPt8j4emm7rdMWQyL0F89o92q10gvCUa6sBkBtDJ7nV2+P7wpXczzOfoDJ49CKXe5CCqb8dc1W+ZdLlrKzSAnQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-7UtIE9eH0u41zpB60Jzr0oNCQ3hMJUabMcKRUVjmyHTXiWDE4vjSqN6qlih7rCNeKGbioS7f/y2Jgym4QZcKFg==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-6hhckcBqVgjWAqLy2vqlPZ3rfxLDhFWEmM7oLh2POGvsi7j0tHkbN7w4DFhuBExVJAbJ/qqxqZdRY6Fu7/OezQ==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-p7iYAPaQjoeM+AKABpYWeDdtwQNxasr4aXQEA/OmbOaug9V0odRVDy3Wx4ci8soljE/JXQo+abV0qZpW8NX0yA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-TCEhLnY581YJ+g1x0hapPz13JFqzmh/pMWL2KEFASC51qCfw3+Y47MrTmea4bUE5vsdxQ4F6/KFbUeSz22Q1ow==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-R3ufuzJRxSJbE58K9AEnL/uSZyVdHzud9wLS8tIbXclxKzoe09CRohj2xV8wpx5tj7ZbiJaKYcutMm1eYgz/0A==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-/jL/V1xdVRt5XppwiaEU8Etp5WHZj609n0xMTuehmCqdoOFbId1M+aEeDWZsQ+8JbEB/BJ6ynY2SlYmOaKtt8w==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-p/MYV+JmqmPyjdgyN2UxAeYDj9cBqCjp0C/NsTWnnjoZUVqoeZ6IrW915L9CAKWVECgv9lVQGc4u/yz26/bI1A==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==}
+ engines: {node: '>=16.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-d5yGlQtmN/z5eoTtIYgkvOw27US2Ous4VycnXatyoImIF9tzlcpnKqQ/V7qhvJmb2p6xZne1NopCLakdTnkBBQ==}
+ engines: {node: '>=16.0.0'}
+
'@stdlib/[email protected]':
resolution: {integrity: sha512-3YHwSWiUqGlTLSwxAWxrqaD1PkgcJniGyotJeIt5X0tSNmSW0/c9RWroCImTUUB3zBkyBJ79MyU9Nf4Qgm59fQ==}
engines: {node: '>=0.10.0', npm: '>2.7.0'}
@@ -2004,6 +2365,9 @@ packages:
resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==}
+ resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==}
+
resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==}
@@ -2146,6 +2510,9 @@ packages:
resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}
engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}
+ resolution: {integrity: sha512-hSo2/tFV29whjFFtVtdS7kHmtUsjfMO1sgwE/d5bhOE4O7Vkj5G1R9lLIqkIprp/+rrvCq3HGvEaOgry7xRcDA==}
+
resolution: {integrity: sha512-D116FKTuauCShJjPuOAFnkyAPMhV/6f403+yPZwyyFY6gErK1AA41y9rQdBvj8eHDZ9sXVJ6TzmzObVfAFh3ig==}
@@ -2565,6 +2932,10 @@ packages:
resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}
+ resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==}
+ hasBin: true
+
resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==}
@@ -2613,6 +2984,10 @@ packages:
resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==}
+ resolution: {integrity: sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==}
+ engines: {node: '>= 18'}
+
resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
engines: {node: '>= 6'}
@@ -2621,6 +2996,10 @@ packages:
resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==}
engines: {node: '>= 12.20'}
+ resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==}
+ engines: {node: '>= 18'}
+
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
engines: {node: '>=12.20.0'}
@@ -3107,6 +3486,9 @@ packages:
resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==}
+ resolution: {integrity: sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==}
+
resolution: {integrity: sha512-ESjNzSlt/sWE8sciZH8kBF8BPlwXPwhR6pWKAw8bw4Bwj+iZcnKW6ONWUutJ7eObuBZQpiIb8S7OYspWrKt7rA==}
engines: {node: '>=14'}
@@ -4027,6 +4409,10 @@ packages:
resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==}
engines: {node: '>=0.6'}
+ resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==}
+ engines: {node: '>=0.6'}
+
resolution: {integrity: sha512-x+NLUpx9SYrcwXtX7ob1gnkSems4i/mGZX5SlYxwIau6RrUSODO89TR/XDGGpn5RPWSYIB+aSfuSlV5+CmbTBg==}
engines: {node: '>=0.6'}
@@ -4365,6 +4751,9 @@ packages:
resolution: {integrity: sha512-syeEEd112om/waJ5gOQ+SaYi+setuidQ4ZIPiQREF4yJeegXhn2HKy6C0JYm7uhVQKfMAvuZ22dIRsnoDv7AMw==}
engines: {node: '>=12.*'}
+ resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==}
+
resolution: {integrity: sha512-h2J6kKEikXnZyurUcCYg215qkQpINOhdWkiclHcWAuVeqXsNrfrYaf1s0qbbcdRyMtrVW48I+VdVTw71Cnn20Q==}
engines: {npm: '>=8'}
@@ -4526,8 +4915,8 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
- resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}
+ resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==}
engines: {node: '>=14.17'}
hasBin: true
@@ -4797,6 +5186,503 @@ snapshots:
transitivePeerDependencies:
- encoding
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/util': 3.0.0
+ '@aws-sdk/types': 3.679.0
+ tslib: 1.14.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-crypto/supports-web-crypto': 5.2.0
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-locate-window': 3.679.0
+ '@smithy/util-utf8': 2.3.0
+ tslib: 2.6.3
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/types': 3.679.0
+ tslib: 2.6.3
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-utf8-browser': 3.259.0
+ tslib: 1.14.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/util-utf8': 2.3.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-crypto/sha256-browser': 5.2.0
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])
+ '@aws-sdk/client-sts': 3.679.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@aws-sdk/middleware-host-header': 3.679.0
+ '@aws-sdk/middleware-logger': 3.679.0
+ '@aws-sdk/middleware-recursion-detection': 3.679.0
+ '@aws-sdk/middleware-user-agent': 3.679.0
+ '@aws-sdk/region-config-resolver': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-endpoints': 3.679.0
+ '@aws-sdk/util-user-agent-browser': 3.679.0
+ '@aws-sdk/util-user-agent-node': 3.679.0
+ '@smithy/config-resolver': 3.0.10
+ '@smithy/core': 2.5.1
+ '@smithy/fetch-http-handler': 3.2.9
+ '@smithy/hash-node': 3.0.8
+ '@smithy/invalid-dependency': 3.0.8
+ '@smithy/middleware-content-length': 3.0.10
+ '@smithy/middleware-endpoint': 3.2.1
+ '@smithy/middleware-retry': 3.0.25
+ '@smithy/middleware-serde': 3.0.8
+ '@smithy/middleware-stack': 3.0.8
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/node-http-handler': 3.2.5
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/url-parser': 3.0.8
+ '@smithy/util-base64': 3.0.0
+ '@smithy/util-body-length-browser': 3.0.0
+ '@smithy/util-body-length-node': 3.0.0
+ '@smithy/util-defaults-mode-browser': 3.0.25
+ '@smithy/util-defaults-mode-node': 3.0.25
+ '@smithy/util-endpoints': 2.1.4
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-retry': 3.0.8
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - aws-crt
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-crypto/sha256-browser': 5.2.0
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])
+ '@aws-sdk/client-sts': 3.679.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@aws-sdk/middleware-host-header': 3.679.0
+ '@aws-sdk/middleware-logger': 3.679.0
+ '@aws-sdk/middleware-recursion-detection': 3.679.0
+ '@aws-sdk/middleware-user-agent': 3.679.0
+ '@aws-sdk/region-config-resolver': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-endpoints': 3.679.0
+ '@aws-sdk/util-user-agent-browser': 3.679.0
+ '@aws-sdk/util-user-agent-node': 3.679.0
+ '@smithy/config-resolver': 3.0.10
+ '@smithy/core': 2.5.1
+ '@smithy/fetch-http-handler': 3.2.9
+ '@smithy/hash-node': 3.0.8
+ '@smithy/invalid-dependency': 3.0.8
+ '@smithy/middleware-content-length': 3.0.10
+ '@smithy/middleware-endpoint': 3.2.1
+ '@smithy/middleware-retry': 3.0.25
+ '@smithy/middleware-serde': 3.0.8
+ '@smithy/middleware-stack': 3.0.8
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/node-http-handler': 3.2.5
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/url-parser': 3.0.8
+ '@smithy/util-base64': 3.0.0
+ '@smithy/util-body-length-browser': 3.0.0
+ '@smithy/util-body-length-node': 3.0.0
+ '@smithy/util-defaults-mode-browser': 3.0.25
+ '@smithy/util-defaults-mode-node': 3.0.25
+ '@smithy/util-endpoints': 2.1.4
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-retry': 3.0.8
+ '@smithy/util-utf8': 3.0.0
+ '@smithy/util-waiter': 3.1.7
+ '@types/uuid': 9.0.8
+ tslib: 2.6.3
+ uuid: 9.0.1
+ transitivePeerDependencies:
+ - aws-crt
+
+ '@aws-sdk/[email protected](@aws-sdk/[email protected])':
+ dependencies:
+ '@aws-crypto/sha256-browser': 5.2.0
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-sdk/client-sts': 3.679.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@aws-sdk/middleware-host-header': 3.679.0
+ '@aws-sdk/middleware-logger': 3.679.0
+ '@aws-sdk/middleware-recursion-detection': 3.679.0
+ '@aws-sdk/middleware-user-agent': 3.679.0
+ '@aws-sdk/region-config-resolver': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-endpoints': 3.679.0
+ '@aws-sdk/util-user-agent-browser': 3.679.0
+ '@aws-sdk/util-user-agent-node': 3.679.0
+ '@smithy/config-resolver': 3.0.10
+ '@smithy/core': 2.5.1
+ '@smithy/fetch-http-handler': 3.2.9
+ '@smithy/hash-node': 3.0.8
+ '@smithy/invalid-dependency': 3.0.8
+ '@smithy/middleware-content-length': 3.0.10
+ '@smithy/middleware-endpoint': 3.2.1
+ '@smithy/middleware-retry': 3.0.25
+ '@smithy/middleware-serde': 3.0.8
+ '@smithy/middleware-stack': 3.0.8
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/node-http-handler': 3.2.5
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/url-parser': 3.0.8
+ '@smithy/util-base64': 3.0.0
+ '@smithy/util-body-length-browser': 3.0.0
+ '@smithy/util-body-length-node': 3.0.0
+ '@smithy/util-defaults-mode-browser': 3.0.25
+ '@smithy/util-defaults-mode-node': 3.0.25
+ '@smithy/util-endpoints': 2.1.4
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-retry': 3.0.8
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - aws-crt
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-crypto/sha256-browser': 5.2.0
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/middleware-host-header': 3.679.0
+ '@aws-sdk/middleware-logger': 3.679.0
+ '@aws-sdk/middleware-recursion-detection': 3.679.0
+ '@aws-sdk/middleware-user-agent': 3.679.0
+ '@aws-sdk/region-config-resolver': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-endpoints': 3.679.0
+ '@aws-sdk/util-user-agent-browser': 3.679.0
+ '@aws-sdk/util-user-agent-node': 3.679.0
+ '@smithy/config-resolver': 3.0.10
+ '@smithy/core': 2.5.1
+ '@smithy/fetch-http-handler': 3.2.9
+ '@smithy/hash-node': 3.0.8
+ '@smithy/invalid-dependency': 3.0.8
+ '@smithy/middleware-content-length': 3.0.10
+ '@smithy/middleware-endpoint': 3.2.1
+ '@smithy/middleware-retry': 3.0.25
+ '@smithy/middleware-serde': 3.0.8
+ '@smithy/middleware-stack': 3.0.8
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/node-http-handler': 3.2.5
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/url-parser': 3.0.8
+ '@smithy/util-base64': 3.0.0
+ '@smithy/util-body-length-browser': 3.0.0
+ '@smithy/util-body-length-node': 3.0.0
+ '@smithy/util-defaults-mode-browser': 3.0.25
+ '@smithy/util-defaults-mode-node': 3.0.25
+ '@smithy/util-endpoints': 2.1.4
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-retry': 3.0.8
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - aws-crt
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-crypto/sha256-browser': 5.2.0
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@aws-sdk/middleware-host-header': 3.679.0
+ '@aws-sdk/middleware-logger': 3.679.0
+ '@aws-sdk/middleware-recursion-detection': 3.679.0
+ '@aws-sdk/middleware-user-agent': 3.679.0
+ '@aws-sdk/region-config-resolver': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-endpoints': 3.679.0
+ '@aws-sdk/util-user-agent-browser': 3.679.0
+ '@aws-sdk/util-user-agent-node': 3.679.0
+ '@smithy/config-resolver': 3.0.10
+ '@smithy/core': 2.5.1
+ '@smithy/fetch-http-handler': 3.2.9
+ '@smithy/hash-node': 3.0.8
+ '@smithy/invalid-dependency': 3.0.8
+ '@smithy/middleware-content-length': 3.0.10
+ '@smithy/middleware-endpoint': 3.2.1
+ '@smithy/middleware-retry': 3.0.25
+ '@smithy/middleware-serde': 3.0.8
+ '@smithy/middleware-stack': 3.0.8
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/node-http-handler': 3.2.5
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/url-parser': 3.0.8
+ '@smithy/util-base64': 3.0.0
+ '@smithy/util-body-length-browser': 3.0.0
+ '@smithy/util-body-length-node': 3.0.0
+ '@smithy/util-defaults-mode-browser': 3.0.25
+ '@smithy/util-defaults-mode-node': 3.0.25
+ '@smithy/util-endpoints': 2.1.4
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-retry': 3.0.8
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - aws-crt
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/core': 2.5.1
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/property-provider': 3.1.8
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/signature-v4': 4.2.1
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/util-middleware': 3.0.8
+ fast-xml-parser: 4.4.1
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/client-cognito-identity': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@smithy/property-provider': 3.1.8
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - aws-crt
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@smithy/property-provider': 3.1.8
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@smithy/fetch-http-handler': 3.2.9
+ '@smithy/node-http-handler': 3.2.5
+ '@smithy/property-provider': 3.1.8
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/util-stream': 3.2.1
+ tslib: 2.6.3
+
+ dependencies:
+ '@aws-sdk/client-sts': 3.679.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/credential-provider-env': 3.679.0
+ '@aws-sdk/credential-provider-http': 3.679.0
+ '@aws-sdk/credential-provider-process': 3.679.0
+ '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
+ '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/[email protected])
+ '@aws-sdk/types': 3.679.0
+ '@smithy/credential-provider-imds': 3.2.5
+ '@smithy/property-provider': 3.1.8
+ '@smithy/shared-ini-file-loader': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - '@aws-sdk/client-sso-oidc'
+ - aws-crt
+
+ dependencies:
+ '@aws-sdk/credential-provider-env': 3.679.0
+ '@aws-sdk/credential-provider-http': 3.679.0
+ '@aws-sdk/credential-provider-ini': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@aws-sdk/credential-provider-process': 3.679.0
+ '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
+ '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/[email protected])
+ '@aws-sdk/types': 3.679.0
+ '@smithy/credential-provider-imds': 3.2.5
+ '@smithy/property-provider': 3.1.8
+ '@smithy/shared-ini-file-loader': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - '@aws-sdk/client-sso-oidc'
+ - '@aws-sdk/client-sts'
+ - aws-crt
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@smithy/property-provider': 3.1.8
+ '@smithy/shared-ini-file-loader': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))':
+ dependencies:
+ '@aws-sdk/client-sso': 3.679.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/token-providers': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
+ '@aws-sdk/types': 3.679.0
+ '@smithy/property-provider': 3.1.8
+ '@smithy/shared-ini-file-loader': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - '@aws-sdk/client-sso-oidc'
+ - aws-crt
+
+ '@aws-sdk/[email protected](@aws-sdk/[email protected])':
+ dependencies:
+ '@aws-sdk/client-sts': 3.679.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@smithy/property-provider': 3.1.8
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))':
+ dependencies:
+ '@aws-sdk/client-cognito-identity': 3.679.0
+ '@aws-sdk/client-sso': 3.679.0
+ '@aws-sdk/client-sts': 3.679.0
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/credential-provider-cognito-identity': 3.679.0
+ '@aws-sdk/credential-provider-env': 3.679.0
+ '@aws-sdk/credential-provider-http': 3.679.0
+ '@aws-sdk/credential-provider-ini': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@aws-sdk/credential-provider-process': 3.679.0
+ '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
+ '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/[email protected])
+ '@aws-sdk/types': 3.679.0
+ '@smithy/credential-provider-imds': 3.2.5
+ '@smithy/property-provider': 3.1.8
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+ transitivePeerDependencies:
+ - '@aws-sdk/client-sso-oidc'
+ - aws-crt
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@aws-sdk/util-endpoints': 3.679.0
+ '@smithy/core': 2.5.1
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@smithy/protocol-http': 1.2.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/types': 3.6.0
+ '@smithy/util-config-provider': 3.0.0
+ '@smithy/util-middleware': 3.0.8
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@smithy/signature-v4': 1.1.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))':
+ dependencies:
+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])
+ '@aws-sdk/types': 3.679.0
+ '@smithy/property-provider': 3.1.8
+ '@smithy/shared-ini-file-loader': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/types': 3.6.0
+ '@smithy/util-endpoints': 2.1.4
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.679.0
+ '@smithy/types': 3.6.0
+ bowser: 2.11.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/middleware-user-agent': 3.679.0
+ '@aws-sdk/types': 3.679.0
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
dependencies:
'@babel/highlight': 7.24.6
@@ -5040,7 +5926,7 @@ snapshots:
'@devil7softwares/[email protected]': {}
- '@dqbd/[email protected]': {}
+ '@dqbd/[email protected]': {}
'@flydotio/[email protected]':
dependencies:
@@ -5338,13 +6224,13 @@ snapshots:
'@js-sdsl/[email protected]': {}
dependencies:
ansi-styles: 5.2.0
camelcase: 6.3.0
decamelize: 1.2.0
js-tiktoken: 1.0.12
+ langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu)
ml-distance: 4.0.1
mustache: 4.2.0
p-queue: 6.6.2
@@ -5356,9 +6242,9 @@ snapshots:
- langchain
- openai
dependencies:
js-tiktoken: 1.0.12
openai: 4.57.0([email protected])
zod: 3.23.8
@@ -5367,9 +6253,9 @@ snapshots:
- encoding
- langchain
dependencies:
js-tiktoken: 1.0.12
transitivePeerDependencies:
- langchain
@@ -6495,6 +7381,340 @@ snapshots:
dependencies:
'@sinonjs/commons': 3.0.1
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/types': 3.6.0
+ '@smithy/util-config-provider': 3.0.0
+ '@smithy/util-middleware': 3.0.8
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/middleware-serde': 3.0.8
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/types': 3.6.0
+ '@smithy/util-body-length-browser': 3.0.0
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-stream': 3.2.1
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/property-provider': 3.1.8
+ '@smithy/types': 3.6.0
+ '@smithy/url-parser': 3.0.8
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@aws-crypto/crc32': 3.0.0
+ '@smithy/types': 1.2.0
+ '@smithy/util-hex-encoding': 1.1.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/querystring-builder': 3.0.8
+ '@smithy/types': 3.6.0
+ '@smithy/util-base64': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/querystring-builder': 3.0.8
+ '@smithy/types': 3.6.0
+ '@smithy/util-base64': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ '@smithy/util-buffer-from': 3.0.0
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/core': 2.5.1
+ '@smithy/middleware-serde': 3.0.8
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/shared-ini-file-loader': 3.1.9
+ '@smithy/types': 3.6.0
+ '@smithy/url-parser': 3.0.8
+ '@smithy/util-middleware': 3.0.8
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/service-error-classification': 3.0.8
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-retry': 3.0.8
+ tslib: 2.6.3
+ uuid: 9.0.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/property-provider': 3.1.8
+ '@smithy/shared-ini-file-loader': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/abort-controller': 3.1.6
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/querystring-builder': 3.0.8
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 1.2.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ '@smithy/util-uri-escape': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/eventstream-codec': 1.1.0
+ '@smithy/is-array-buffer': 1.1.0
+ '@smithy/types': 1.2.0
+ '@smithy/util-hex-encoding': 1.1.0
+ '@smithy/util-middleware': 1.1.0
+ '@smithy/util-uri-escape': 1.1.0
+ '@smithy/util-utf8': 1.1.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/is-array-buffer': 3.0.0
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/types': 3.6.0
+ '@smithy/util-hex-encoding': 3.0.0
+ '@smithy/util-middleware': 3.0.8
+ '@smithy/util-uri-escape': 3.0.0
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/core': 2.5.1
+ '@smithy/middleware-endpoint': 3.2.1
+ '@smithy/middleware-stack': 3.0.8
+ '@smithy/protocol-http': 4.1.5
+ '@smithy/types': 3.6.0
+ '@smithy/util-stream': 3.2.1
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/querystring-parser': 3.0.8
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/util-buffer-from': 3.0.0
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/is-array-buffer': 1.1.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/is-array-buffer': 2.2.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/is-array-buffer': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/property-provider': 3.1.8
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ bowser: 2.11.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/config-resolver': 3.0.10
+ '@smithy/credential-provider-imds': 3.2.5
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/property-provider': 3.1.8
+ '@smithy/smithy-client': 3.4.2
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/node-config-provider': 3.1.9
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/service-error-classification': 3.0.8
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/fetch-http-handler': 4.0.0
+ '@smithy/node-http-handler': 3.2.5
+ '@smithy/types': 3.6.0
+ '@smithy/util-base64': 3.0.0
+ '@smithy/util-buffer-from': 3.0.0
+ '@smithy/util-hex-encoding': 3.0.0
+ '@smithy/util-utf8': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/util-buffer-from': 1.1.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/util-buffer-from': 2.2.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/util-buffer-from': 3.0.0
+ tslib: 2.6.3
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/abort-controller': 3.1.6
+ '@smithy/types': 3.6.0
+ tslib: 2.6.3
+
'@stdlib/[email protected]': {}
'@stdlib/[email protected]':
@@ -7106,6 +8326,8 @@ snapshots:
+
dependencies:
balanced-match: 1.0.2
@@ -7263,6 +8485,25 @@ snapshots:
+ dependencies:
+ '@aws-sdk/client-sagemaker': 3.679.0
+ '@aws-sdk/credential-providers': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
+ '@aws-sdk/protocol-http': 3.374.0
+ '@aws-sdk/signature-v4': 3.374.0
+ form-data: 4.0.0
+ form-data-encoder: 4.0.2
+ formdata-node: 6.0.3
+ js-base64: 3.7.2
+ node-fetch: 2.7.0
+ qs: 6.11.2
+ readable-stream: 4.5.2
+ url-join: 4.0.1
+ transitivePeerDependencies:
+ - '@aws-sdk/client-sso-oidc'
+ - aws-crt
+ - encoding
+
@@ -7650,6 +8891,10 @@ snapshots:
+ dependencies:
+ strnum: 1.0.5
+
dependencies:
bser: 2.1.1
@@ -7701,6 +8946,8 @@ snapshots:
+
dependencies:
asynckit: 0.4.0
@@ -7712,6 +8959,8 @@ snapshots:
node-domexception: 1.0.0
web-streams-polyfill: 4.0.0-beta.3
+
dependencies:
fetch-blob: 3.2.0
@@ -8447,6 +9696,8 @@ snapshots:
+
dependencies:
config-chain: 1.1.13
@@ -8513,17 +9764,17 @@ snapshots:
dependencies:
binary-extensions: 2.3.0
js-tiktoken: 1.0.12
js-yaml: 4.1.0
jsonpointer: 5.0.1
langchainhub: 0.0.11
+ langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu)
ml-distance: 4.0.1
openapi-types: 12.1.3
p-retry: 4.6.2
@@ -8532,14 +9783,16 @@ snapshots:
zod: 3.23.8
zod-to-json-schema: 3.23.1([email protected])
optionalDependencies:
+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
'@supabase/supabase-js': 2.44.2
axios: 1.7.2
cheerio: 1.0.0-rc.12
+ fast-xml-parser: 4.4.1
handlebars: 4.7.8
html-to-text: 9.0.5
ioredis: 5.4.1
mammoth: 1.7.2
- mongodb: 6.6.2([email protected])
+ mongodb: 6.6.2(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])
pdf-parse: 1.1.1
puppeteer: 22.12.1([email protected])
redis: 4.6.14
@@ -8550,7 +9803,7 @@ snapshots:
+ [email protected](zyeavx4tfqw3smbbpiinhfxxeu):
dependencies:
'@types/uuid': 9.0.8
commander: 10.0.1
@@ -8559,8 +9812,8 @@ snapshots:
p-retry: 4.6.2
uuid: 9.0.1
optionalDependencies:
openai: 4.57.0([email protected])
@@ -8768,19 +10021,20 @@ snapshots:
'@types/whatwg-url': 11.0.5
whatwg-url: 13.0.0
dependencies:
'@mongodb-js/saslprep': 1.1.7
bson: 6.8.0
mongodb-connection-string-url: 3.0.1
optionalDependencies:
+ '@aws-sdk/credential-providers': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
socks: 2.8.3
dependencies:
bson: 6.8.0
kareem: 2.6.3
- mongodb: 6.6.2([email protected])
+ mongodb: 6.6.2(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])
mpath: 0.9.0
mquery: 5.0.0
ms: 2.1.3
@@ -8829,7 +10083,7 @@ snapshots:
dependencies:
afinn-165: 1.0.4
afinn-165-financialmarketnews: 3.0.0
@@ -8837,7 +10091,7 @@ snapshots:
dotenv: 16.4.5
http-server: 14.1.1
memjs: 1.3.2
- mongoose: 8.4.4([email protected])
+ mongoose: 8.4.4(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])
pg: 8.12.0
redis: 4.6.14
safe-stable-stringify: 2.4.3
@@ -9208,7 +10462,7 @@ snapshots:
csv-parse: 5.5.6
gpt3-tokenizer: 1.1.5
openai: 3.3.0
- typescript: 5.6.2
+ typescript: 5.6.3
uuid: 9.0.1
zod: 3.23.8
transitivePeerDependencies:
@@ -9295,6 +10549,10 @@ snapshots:
dependencies:
side-channel: 1.0.6
+ dependencies:
+ side-channel: 1.0.6
+
dependencies:
side-channel: 1.0.6
@@ -9643,6 +10901,8 @@ snapshots:
'@types/node': 20.14.1
qs: 6.12.2
+
dependencies:
bin-links: 4.0.4
@@ -9806,7 +11066,7 @@ snapshots:
dependencies:
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
new file mode 100644
index 000000000..f4e93e7fc
--- /dev/null
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -0,0 +1,197 @@
+import { Request, Response } from "express";
+import { Logger } from "../../lib/logger";
+import {
+ Document,
+ legacyDocumentConverter,
+ legacyExtractorOptions,
+ legacyScrapeOptions,
+ RequestWithAuth,
+ ExtractRequest,
+ extractRequestSchema,
+ ExtractResponse,
+ legacyCrawlerOptions,
+ MapDocument,
+} from "./types";
+import { billTeam } from "../../services/billing/credit_billing";
+import { v4 as uuidv4 } from "uuid";
+import { numTokensFromString } from "../../lib/LLM-extraction/helpers";
+import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
+import { logJob } from "../../services/logging/log_job";
+import { getJobPriority } from "../../lib/job-priority";
+import { PlanType } from "../../types";
+import { getMapResults } from "./map";
+import { rerankDocuments } from "../../lib/extract/reranker";
+import { generateBasicCompletion } from "../../lib/extract/completions";
+
+
+
+export async function extractController(
+ req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
+ res: Response<ExtractResponse>
+) {
+ req.body = extractRequestSchema.parse(req.body);
+ let earlyReturn = false;
+
+ const origin = req.body.origin;
+ const timeout = req.body.timeout;
+// const pageOptions = legacyScrapeOptions(req.body);
+// const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;
+ const jobId = uuidv4();
+
+ const startTime = new Date().getTime();
+ const jobPriority = await getJobPriority({
+ plan: req.auth.plan as PlanType,
+ team_id: req.auth.team_id,
+ basePriority: 10,
+ });
+
+ const urls = req.body.urls;
+ const mappedDocuments: MapDocument[] = [];
+
+ const prompt = req.body.prompt;
+ const keywords = await generateBasicCompletion(`If the user's prompt is: "${prompt}", what are the most important keywords besides the extraction task? Output only the keywords, separated by commas.`);
+
+ for (const url of urls) {
+ if (url.endsWith("/*")) {
+ const mapResults = await getMapResults({
+ url: url.slice(0, -2),
+ search: req.body.prompt,
+ limit: 100,
+ ignoreSitemap: true,
+ includeSubdomains: false,
+ crawlerOptions: {},
+ teamId: req.auth.team_id,
+ plan: req.auth.plan,
+ origin: req.body.origin,
+ subId: req.acuc?.sub_id,
+ includeMetadata: true
+ });
+ // top 3 links
+ const top3Links = (mapResults.links as MapDocument[]).slice(0, 3);
+ console.log(top3Links);
+ // console.log(top3Links);
+ mappedDocuments.push(...(mapResults.links as MapDocument[]));
+ // transform mappedUrls to just documents
+ // we quickly rerank
+ const rerank = await rerankDocuments(mappedDocuments.map(x => `URL: ${x.url}\nTITLE: ${x.title}\nDESCRIPTION: ${x.description}`), "What URLs are most relevant to the following prompt: " + req.body.prompt.toLocaleLowerCase().replace("extract", " ").replace("extract ", " "));
+ console.log(rerank);
+ } else {
+ mappedDocuments.push({ url });
+ }
+ }
+
+ req.body.urls = mappedDocuments.map(x => x.url);
+
+
+
+// const job = await addScrapeJob(
+// {
+// url: req.body.url,
+// mode: "single_urls",
+// crawlerOptions: {},
+// team_id: req.auth.team_id,
+// plan: req.auth.plan,
+// pageOptions,
+// extractorOptions,
+// origin: req.body.origin,
+// is_scrape: true,
+// },
+// {},
+// jobId,
+// jobPriority
+// );
+
+// const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds : 0) + a, 0);
+
+// let doc: any | undefined;
+// try {
+// doc = (await waitForJob(job.id, timeout + totalWait))[0];
+// } catch (e) {
+// Logger.error(`Error in scrapeController: ${e}`);
+// if (e instanceof Error && e.message.startsWith("Job wait")) {
+// return res.status(408).json({
+// success: false,
+// error: "Request timed out",
+// });
+// } else {
+// return res.status(500).json({
+// success: false,
+// error: `(Internal server error) - ${e && e?.message ? e.message : e} ${
+// extractorOptions && extractorOptions.mode !== "markdown"
+// ? " - Could be due to LLM parsing issues"
+// : ""
+// }`,
+// });
+// }
+// }
+
+// await job.remove();
+
+// if (!doc) {
+// console.error("!!! PANIC DOC IS", doc, job);
+// return res.status(200).json({
+// success: true,
+// warning: "No page found",
+// data: doc,
+// });
+// }
+
+// delete doc.index;
+// delete doc.provider;
+
+// const endTime = new Date().getTime();
+// const timeTakenInSeconds = (endTime - startTime) / 1000;
+// const numTokens =
+// doc && doc.markdown
+// ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
+// : 0;
+
+// let creditsToBeBilled = 1; // Assuming 1 credit per document
+// if (earlyReturn) {
+// // Don't bill if we're early returning
+// return;
+// }
+// if(req.body.extract && req.body.formats.includes("extract")) {
+// creditsToBeBilled = 5;
+// }
+
+// billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(error => {
+// Logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`);
+// // Optionally, you could notify an admin or add to a retry queue here
+// });
+
+// if (!pageOptions || !pageOptions.includeRawHtml) {
+// if (doc && doc.rawHtml) {
+// delete doc.rawHtml;
+// }
+// }
+
+// if(pageOptions && pageOptions.includeExtract) {
+// if(!pageOptions.includeMarkdown && doc && doc.markdown) {
+// delete doc.markdown;
+// }
+// }
+
+// logJob({
+// job_id: jobId,
+// success: true,
+// message: "Scrape completed",
+// num_docs: 1,
+// docs: [doc],
+// time_taken: timeTakenInSeconds,
+// team_id: req.auth.team_id,
+// mode: "scrape",
+// url: req.body.url,
+// crawlerOptions: {},
+// pageOptions: pageOptions,
+// origin: origin,
+// extractor_options: extractorOptions,
+// num_tokens: numTokens,
+// });
+
+ return res.status(200).json({
+ success: true,
+ data: null,
+ scrape_id: origin?.includes("website") ? jobId : undefined,
+ });
+}
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 5ed3dd51c..7e74b43e4 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -15,11 +15,11 @@ import {
removeDuplicateUrls,
} from "../../lib/validateUrl";
import { fireEngineMap } from "../../search/fireEngine";
-import { billTeam } from "../../services/billing/credit_billing";
-import { logJob } from "../../services/logging/log_job";
import { performCosineSimilarity } from "../../lib/map-cosine";
import { Logger } from "../../lib/logger";
import Redis from "ioredis";
+import { billTeam } from "../../services/billing/credit_billing";
+import { logJob } from "../../services/logging/log_job";
configDotenv();
const redis = new Redis(process.env.REDIS_URL);
@@ -29,35 +29,50 @@ const MAX_MAP_LIMIT = 5000;
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 1000;
-export async function mapController(
- req: RequestWithAuth<{}, MapResponse, MapRequest>,
- res: Response<MapResponse>
-) {
- const startTime = new Date().getTime();
-
- req.body = mapRequestSchema.parse(req.body);
-
- const limit: number = req.body.limit ?? MAX_MAP_LIMIT;
+interface MapOptions {
+ url: string;
+ search?: string;
+ limit?: number;
+ ignoreSitemap?: boolean;
+ includeSubdomains?: boolean;
+ crawlerOptions?: any;
+ teamId: string;
+ plan: string;
+ origin?: string;
+ subId?: string;
+ includeMetadata?: boolean;
+}
+export async function getMapResults({
+ url,
+ search,
+ limit = MAX_MAP_LIMIT,
+ ignoreSitemap = false,
+ includeSubdomains = false,
+ crawlerOptions = {},
+ teamId,
+ plan,
+ origin,
+ subId,
+ includeMetadata = false,
+}: MapOptions) {
+ const startTime = new Date().getTime();
const id = uuidv4();
- let links: string[] = [req.body.url];
+ let links: { url: string; title?: string; description?: string }[] = [{ url }];
const sc: StoredCrawl = {
- originUrl: req.body.url,
- crawlerOptions: legacyCrawlerOptions(req.body),
+ originUrl: url,
+ crawlerOptions,
pageOptions: {},
- team_id: req.auth.team_id,
+ team_id: teamId,
createdAt: Date.now(),
- plan: req.auth.plan,
+ plan,
};
const crawler = crawlToCrawler(id, sc);
- let urlWithoutWww = req.body.url.replace("www.", "");
-
- let mapUrl = req.body.search
- ? `"${req.body.search}" site:${urlWithoutWww}`
- : `site:${req.body.url}`;
+ let urlWithoutWww = url.replace("www.", "");
+ let mapUrl = search ? `"${search}" site:${urlWithoutWww}` : `site:${url}`;
const resultsPerPage = 100;
const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);
@@ -81,12 +96,11 @@ export async function mapController(
pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1));
allResults = await Promise.all(pagePromises);
- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours
+ await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60);
}
- // Parallelize sitemap fetch with serper search
const [sitemap, ...searchResults] = await Promise.all([
- req.body.ignoreSitemap ? null : crawler.tryGetSitemap(),
+ ignoreSitemap ? null : crawler.tryGetSitemap(),
...(cachedResult ? [] : pagePromises),
]);
@@ -96,7 +110,7 @@ export async function mapController(
if (sitemap !== null) {
sitemap.forEach((x) => {
- links.push(x.url);
+ links.push({ url: x.url });
});
}
@@ -110,67 +124,96 @@ export async function mapController(
}
if (mapResults.length > 0) {
- if (req.body.search) {
- // Ensure all map results are first, maintaining their order
+ if (search) {
links = [
- mapResults[0].url,
- ...mapResults.slice(1).map((x) => x.url),
+ { url: mapResults[0].url, title: mapResults[0].title, description: mapResults[0].description },
+ ...mapResults.slice(1).map((x) => ({
+ url: x.url,
+ title: x.title,
+ description: x.description
+ })),
...links,
];
} else {
- mapResults.map((x) => {
- links.push(x.url);
+ mapResults.forEach((x) => {
+ links.push({
+ url: x.url,
+ title: x.title,
+ description: x.description
+ });
});
}
}
- // Perform cosine similarity between the search query and the list of links
- if (req.body.search) {
- const searchQuery = req.body.search.toLowerCase();
-
- links = performCosineSimilarity(links, searchQuery);
+ if (search) {
+ const filteredLinks = performCosineSimilarity(links.map(l => l.url), search.toLowerCase());
+ links = links.filter(l => filteredLinks.includes(l.url));
}
links = links
.map((x) => {
try {
- return checkAndUpdateURLForMap(x).url.trim();
+ return { ...x, url: checkAndUpdateURLForMap(x.url).url.trim() };
} catch (_) {
return null;
}
})
.filter((x) => x !== null);
- // allows for subdomains to be included
- links = links.filter((x) => isSameDomain(x, req.body.url));
+ links = links.filter((x) => isSameDomain(x.url, url));
- // if includeSubdomains is false, filter out subdomains
- if (!req.body.includeSubdomains) {
- links = links.filter((x) => isSameSubdomain(x, req.body.url));
+ if (!includeSubdomains) {
+ links = links.filter((x) => isSameSubdomain(x.url, url));
}
- // remove duplicates that could be due to http/https or www
- links = removeDuplicateUrls(links);
-
- billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {
- Logger.error(
- `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`
- );
- // Optionally, you could notify an admin or add to a retry queue here
- });
+ links = removeDuplicateUrls(links.map(l => l.url)).map(url => links.find(l => l.url === url));
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - startTime) / 1000;
const linksToReturn = links.slice(0, limit);
- logJob({
- job_id: id,
- success: links.length > 0,
+ return {
+ links: includeMetadata ? linksToReturn : linksToReturn.map(l => l.url),
+ scrapeId: origin?.includes("website") ? id : undefined,
+ timeTakenInSeconds,
+ id,
+ linksLength: links.length,
+ linksToReturnLength: linksToReturn.length,
+ docs: linksToReturn.map(l => l.url),
+ };
+}
+
+export async function mapController(
+ req: RequestWithAuth<{}, MapResponse, MapRequest>,
+ res: Response<MapResponse>
+) {
+ req.body = mapRequestSchema.parse(req.body);
+
+ const results = await getMapResults({
+ url: req.body.url,
+ search: req.body.search,
+ limit: req.body.limit,
+ ignoreSitemap: req.body.ignoreSitemap,
+ includeSubdomains: req.body.includeSubdomains,
+ crawlerOptions: legacyCrawlerOptions(req.body),
+ teamId: req.auth.team_id,
+ plan: req.auth.plan,
+ origin: req.body.origin,
+ subId: req.acuc?.sub_id,
+ });
+
+ await billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {
+ Logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`);
+ });
+
+ await logJob({
+ job_id: results.id,
+ success: results.linksLength > 0,
message: "Map completed",
- num_docs: linksToReturn.length,
- docs: linksToReturn,
- time_taken: timeTakenInSeconds,
+ num_docs: results.linksToReturnLength,
+ docs: results.docs,
+ time_taken: results.timeTakenInSeconds,
team_id: req.auth.team_id,
mode: "map",
url: req.body.url,
@@ -183,55 +226,7 @@ export async function mapController(
return res.status(200).json({
success: true,
- links: linksToReturn,
- scrape_id: req.body.origin?.includes("website") ? id : undefined,
+ links: results.links.map(l => l.url),
+ scrape_id: results.scrapeId,
});
}
-
-// Subdomain sitemap url checking
-
-// // For each result, check for subdomains, get their sitemaps and add them to the links
-// const processedUrls = new Set();
-// const processedSubdomains = new Set();
-
-// for (const result of links) {
-// let url;
-// let hostParts;
-// try {
-// url = new URL(result);
-// hostParts = url.hostname.split('.');
-// } catch (e) {
-// continue;
-// }
-
-// console.log("hostParts", hostParts);
-// // Check if it's a subdomain (more than 2 parts, and not 'www')
-// if (hostParts.length > 2 && hostParts[0] !== 'www') {
-// const subdomain = hostParts[0];
-// console.log("subdomain", subdomain);
-// const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`;
-// console.log("subdomainUrl", subdomainUrl);
-
-// if (!processedSubdomains.has(subdomainUrl)) {
-// processedSubdomains.add(subdomainUrl);
-
-// const subdomainCrawl = crawlToCrawler(id, {
-// originUrl: subdomainUrl,
-// crawlerOptions: legacyCrawlerOptions(req.body),
-// pageOptions: {},
-// team_id: req.auth.team_id,
-// createdAt: Date.now(),
-// plan: req.auth.plan,
-// });
-// const subdomainSitemap = await subdomainCrawl.tryGetSitemap();
-// if (subdomainSitemap) {
-// subdomainSitemap.forEach((x) => {
-// if (!processedUrls.has(x.url)) {
-// processedUrls.add(x.url);
-// links.push(x.url);
-// }
-// });
-// }
-// }
-// }
-// }
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 22ac62947..4e31dd86f 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -121,8 +121,21 @@ export const scrapeOptions = z.object({
}).strict(strictMessage)
+
export type ScrapeOptions = z.infer<typeof scrapeOptions>;
+export const extractV1Options = z.object({
+ urls: url.array(),
+ prompt: z.string().optional(),
+ schema: z.any().optional(),
+ origin: z.string().optional().default("api"),
+ timeout: z.number().int().positive().finite().safe().default(60000),
+}).strict(strictMessage)
+
+export type ExtractV1Options = z.infer<typeof extractV1Options>;
+export const extractRequestSchema = extractV1Options;
+export type ExtractRequest = z.infer<typeof extractRequestSchema>;
+
export const scrapeRequestSchema = scrapeOptions.extend({
url,
origin: z.string().optional().default("api"),
@@ -142,6 +155,8 @@ export const scrapeRequestSchema = scrapeOptions.extend({
return obj;
});
+
+
export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
export const batchScrapeRequestSchema = scrapeOptions.extend({
@@ -296,6 +311,21 @@ export interface ScrapeResponseRequestTest {
error?: string;
}
+export type ExtractResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ warning?: string;
+ data: Document;
+ scrape_id?: string;
+ };
+
+export interface ExtractResponseRequestTest {
+ statusCode: number;
+ body: ExtractResponse;
+ error?: string;
+}
+
export type CrawlResponse =
| ErrorResponse
| {
@@ -492,3 +522,11 @@ export function legacyDocumentConverter(doc: any): Document {
},
};
}
+
+
+
+export interface MapDocument {
+ url: string;
+ title?: string;
+ description?: string;
+}
\ No newline at end of file
diff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.ts
new file mode 100644
index 000000000..230584b4a
--- /dev/null
+++ b/apps/api/src/lib/extract/completions.ts
@@ -0,0 +1,119 @@
+import OpenAI from "openai";
+import { encoding_for_model } from "@dqbd/tiktoken";
+import { TiktokenModel } from "@dqbd/tiktoken";
+import { ExtractOptions } from "../../controllers/v1/types";
+import { Document } from "../entities";
+import { z } from "zod";
+
+const maxTokens = 32000;
+const modifier = 4;
+
+export class LLMRefusalError extends Error {
+ constructor(refusal: string) {
+ super("LLM refused to extract the website's content");
+ this.name = "LLMRefusalError";
+ }
+}
+
+interface GenerateCompletionsParams {
+ systemPrompt?: string;
+ prompt?: string;
+ schema?: any;
+ pagesContent: string;
+}
+
+export async function generateBasicCompletion(prompt: string) {
+ const openai = new OpenAI();
+ const model: TiktokenModel =
+ (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";
+
+ const completion = await openai.chat.completions.create({
+ model,
+ messages: [{ role: "user", content: prompt }],
+ });
+
+ return completion.choices[0].message.content;
+}
+
+export async function generateFinalExtraction({
+ pagesContent,
+ systemPrompt,
+ prompt,
+ schema,
+}: GenerateCompletionsParams): Promise<{
+ content: string;
+ metadata: { numTokens: number; warning: string };
+}> {
+ const openai = new OpenAI();
+ const model: TiktokenModel =
+ (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";
+
+ let extractionContent = pagesContent;
+ let numTokens = 0;
+ let warning = "";
+
+ const encoder = encoding_for_model(model);
+ try {
+ const tokens = encoder.encode(extractionContent);
+ numTokens = tokens.length;
+ } catch (error) {
+ extractionContent = extractionContent.slice(0, maxTokens * modifier);
+ warning = `Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (${maxTokens}) we support.`;
+ } finally {
+ encoder.free();
+ }
+
+ if (numTokens > maxTokens) {
+ extractionContent = extractionContent.slice(0, maxTokens * modifier);
+ warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;
+ }
+
+ if (schema && (schema.type === "array" || schema._type === "ZodArray")) {
+ schema = {
+ type: "object",
+ properties: {
+ items: schema,
+ },
+ required: ["items"],
+ additionalProperties: false,
+ };
+ }
+
+ const jsonCompletion = await openai.beta.chat.completions.parse({
+ model,
+ messages: [
+ { role: "system", content: systemPrompt },
+ { role: "user", content: [{ type: "text", text: extractionContent }] },
+ {
+ role: "user",
+ content: prompt
+ ? `Transform the above content into structured JSON output based on the following user request: ${prompt}`
+ : "Transform the above content into structured JSON output.",
+ },
+ ],
+ response_format: schema
+ ? {
+ type: "json_schema",
+ json_schema: {
+ name: "websiteContent",
+ schema: schema.shape,
+ strict: true,
+ },
+ }
+ : { type: "json_object" },
+ });
+
+ if (jsonCompletion.choices[0].message.refusal !== null) {
+ throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal);
+ }
+
+ const extraction = jsonCompletion.choices[0].message.parsed;
+
+ return {
+ content: extraction,
+ metadata: {
+ numTokens,
+ warning,
+ },
+ };
+}
diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts
new file mode 100644
index 000000000..30aca441a
--- /dev/null
+++ b/apps/api/src/lib/extract/reranker.ts
@@ -0,0 +1,22 @@
+import { CohereClient } from "cohere-ai";
+import { MapDocument } from "../../controllers/v1/types";
+const cohere = new CohereClient({
+ token: process.env.COHERE_API_KEY,
+});
+
+export async function rerankDocuments(
+ documents: (string | Record<string, string>)[],
+ query: string,
+ topN = 3,
+ model = "rerank-english-v3.0"
+) {
+ const rerank = await cohere.v2.rerank({
+ documents,
+ query,
+ topN,
+ model,
+ returnDocuments: true,
+ });
+
+ return rerank.results.sort((a, b) => b.relevanceScore - a.relevanceScore).map(x => ({ document: x.document, index: x.index, relevanceScore: x.relevanceScore }));
+}