Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: wip
Nicolas78badf8f7265
diff --git a/apps/api/package.json b/apps/api/package.jsonindex a0f9cf8e5..dc62d4c35 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -53,7 +53,7 @@"@bull-board/api": "^5.20.5","@bull-board/express": "^5.20.5","@devil7softwares/pos": "^1.0.2",- "@dqbd/tiktoken": "^1.0.13",+ "@dqbd/tiktoken": "^1.0.17","@hyperdx/node-opentelemetry": "^0.8.1","@logtail/node": "^0.4.12","@nangohq/node": "^0.40.8",@@ -73,6 +73,7 @@"cacheable-lookup": "^6.1.0","cheerio": "^1.0.0-rc.12","cohere": "^1.1.1",+ "cohere-ai": "^7.14.0","cors": "^2.8.5","cron-parser": "^4.9.0","date-fns": "^3.6.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 095b507c3..f07c4ecda 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -24,8 +24,8 @@ importers:specifier: ^1.0.2version: 1.0.2'@dqbd/tiktoken':- specifier: ^1.0.13- version: 1.0.15+ specifier: ^1.0.17+ version: 1.0.17'@hyperdx/node-opentelemetry':specifier: ^0.8.1version: 0.8.1@@ -83,6 +83,9 @@ importers:cohere:specifier: ^1.1.1version: 1.1.1+ cohere-ai:+ specifier: ^7.14.0+ version: 7.14.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))cors:specifier: ^2.8.5version: 2.8.5@@ -130,7 +133,7 @@ importers:version: 2.9.0langchain:specifier: ^0.2.8- version: 0.2.8(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])+ version: 0.2.8(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])languagedetect:specifier: ^2.0.0version: 2.0.0@@ -148,10 +151,10 @@ importers:version: 2.30.1mongoose:specifier: ^8.4.4- version: 8.4.4([email protected])+ version: 8.4.4(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])natural:specifier: ^7.0.7- version: 7.0.7([email protected])+ version: 7.0.7(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])openai:specifier: ^4.57.0version: 4.57.0([email protected])@@ -289,6 +292,154 @@ packages:'@anthropic-ai/[email protected]':resolution: {integrity: sha512-916wJXO6T6k8R6BAAcLhLPv/pnLGy7YSEBZXZ1XTFbLcTZE8oTy3oDW9WJf9KKZwMvVcePIfoTSvzXHRcGxkQQ==}+ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==}+ engines: {node: '>=16.0.0'}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-vJzQ6QpaMu8itJMe3FH1/0rwMjL0ELh63iLTxiAmhiV/SvCwNNoSFLd2HdKxbV0Bg/x8lUiPVq3pl6+cxaIrEQ==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-n1hTHpQl6LwNkwn4vLmtbwkNoX2jxtiliRd0IaHR1CfAQvKNTfQ52mARWr73hR+/YcVsBzPx8sYKq2XHWArHKQ==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-/dBYWcCwbA/id4sFCIVZvf0UsvzHCC68SryxeNQk/PDkY9N4n5yRcMUkZDaEyQCjowc3kY4JOXp2AdUP037nhA==}+ engines: {node: '>=16.0.0'}+ peerDependencies:+ '@aws-sdk/client-sts': ^3.679.0++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-/0cAvYnpOZTo/Y961F1kx2fhDDLUYZ0SQQ5/75gh3xVImLj7Zw+vp74ieqFbqWLYGMaq8z1Arr9A8zG95mbLdg==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-3CvrT8w1RjFu1g8vKA5Azfr5V83r2/b68Ock43WE003Bq/5Y38mwmYX7vk0fPHzC3qejt4YMAWk/C3fSKOy25g==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-CS6PWGX8l4v/xyvX8RtXnBisdCa5+URzKd0L6GvHChype9qKUVxO/Gg6N/y43Hvg7MNWJt9FBPNWIxUB+byJwg==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-XvWd6RPk7TA7tmqITT+NXvJ6ltJP8BUtLO1NAvja4HKExPKR9HAyoOeeH7KM3lVRED4e4LUnLb3fzteH20IXaA==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-EdlTYbzMm3G7VUNAMxr9S1nC1qUNqhKlAxFU8E7cKsAe8Bp29CD5HAs3POc56AVo9GC4yRIS+/mtlZSmrckzUA==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-ZoKLubW5DqqV1/2a3TSn+9sSKg0T8SsYMt1JeirnuLJF0mCoYFUaWMyvxxKuxPoqvUsaycxKru4GkpJ10ltNBw==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-Rg7t8RwUzKcumpipG4neZqaeJ6DF+Bco1+FHn5BZB68jpvwvjBjcQUuWkxj18B6ctYHr1fkunnzeKEn/+vy7+w==}+ engines: {node: '>=16.0.0'}+ peerDependencies:+ '@aws-sdk/client-sts': ^3.679.0++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-E3lBtaqCte8tWs6Rkssc8sLzvGoJ10TLGvpkijOlz43wPd6xCRh1YLwg6zolf9fVFtEyUs/GsgymiASOyxhFtw==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-u/p4TV8kQ0zJWDdZD4+vdQFTMhkDEJFws040Gm113VHa/Xo1SYOjbpvqeuFoz6VmM0bLvoOWjxB9MxnSQbwKpQ==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-SAtWonhi9asxn0ukEbcE81jkyanKgqpsrtskvYPpO9Z9KOednM4Cqt6h1bfcS9zaHjN2zu815Gv8O7WiV+F/DQ==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-a74tLccVznXCaBefWPSysUcLXYJiSkeUmQGtalNgJ1vGkE36W5l/8czFiiowdWdKWz7+x6xf0w+Kjkjlj42Ung==}+ engines: {node: '>=16.0.0'}+ peerDependencies:+ '@aws-sdk/client-sts': ^3.679.0++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-ZjZZb6OERw/UKlSqcJ24AUJIf/ekDLPZrPpo0kPMV70EQ0GkBiklIZ8qULu9bEcI2I4UIapBKRiXTrK4gA6YHg==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-y176HuQ8JRY3hGX8rQzHDSbCl9P5Ny9l16z4xmaiLo+Qfte7ee4Yr3yaAKd7GFoJ3/Mhud2XZ37fR015MfYl2w==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-0vet8InEj7nvIvGKk+ch7bEF5SyZ7Us9U7YTEgXPrBNStKeRUsgwRm0ijPWWd0a3oz2okaEwXsFl7G/vI0XiEA==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-sQoAZFsQiW/LL3DfKMYwBoGjYDEnMbA9WslWN8xneCmBAwKo6IcSksvYs23PP8XMIoBGe2I2J9BSr654XWygTQ==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-4hdeXhPDURPqQLPd9jCpUEo9fQITXl3NM3W1MwcJpE0gdUM36uXkQOYsTPeeU/IRCLVjK8Htlh2oCaM9iJrLCA==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-9WpRUbINdGroV3HiZZIBoJvL2ndoWk39OfwxWs2otxByppJZNN14bg/lvCx5e8ggHUti7IBk5rb0nqQZ4m05pg==}+ engines: {node: '>=14.0.0'}+ deprecated: This package has moved to @smithy/protocol-http++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-Ybx54P8Tg6KKq5ck7uwdjiKif7n/8g1x+V0V9uTjBjRWqaIgiqzXwKWoPj6NCNkE7tJNtqI4JrNxp/3S3HvmRw==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-2xLJvSdzcZZAg0lsDLUAuSQuihzK0dcxIK7WmfuJeF7DGKJFmp9czQmz5f3qiDz6IDQzvgK1M9vtJSVCslJbyQ==}+ engines: {node: '>=14.0.0'}+ deprecated: This package has moved to @smithy/signature-v4++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-1/+Zso/x2jqgutKixYFQEGli0FELTgah6bm7aB+m2FAWH4Hz7+iMUsazg6nSWm714sG9G3h5u42Dmpvi9X6/hA==}+ engines: {node: '>=16.0.0'}+ peerDependencies:+ '@aws-sdk/client-sso-oidc': ^3.679.0++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-NwVq8YvInxQdJ47+zz4fH3BRRLC6lL+WLkvr242PVBbUOLRyK/lkwHlfiKUoeVIMyK5NF+up6TRg71t/8Bny6Q==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-YL6s4Y/1zC45OvddvgE139fjeWSKKPgLlnfrvhVL7alNyY9n7beR4uhoDpNrt5mI6sn9qiBF17790o+xLAXjjg==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-zKTd48/ZWrCplkXpYDABI74rQlbR0DNHs8nH95htfSLj9/mWRSwaGptoxwcihaq/77vi/fl2X3y0a1Bo8bt7RA==}+ engines: {node: '>=16.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-CusSm2bTBG1kFypcsqU8COhnYc6zltobsqs3nRrvYqYaOqtMnuE46K4XTWpnzKgwDejgZGOE+WYyprtAxrPvmQ==}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-Bw4uXZ+NU5ed6TNfo4tBbhBSW+2eQxXYjYBGl5gLUNUpg2pDFToQAP6rXBFiwcG52V2ny5oLGiD82SoYuYkAVg==}+ engines: {node: '>=16.0.0'}+ peerDependencies:+ aws-crt: '>=1.0.0'+ peerDependenciesMeta:+ aws-crt:+ optional: true++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==}+'@babel/[email protected]':resolution: {integrity: sha512-ZJhac6FkEd1yhG2AHOmfcXG4ceoLltoCVJjN5XsWN9BifBQr+cHJbWi0h68HZuSORq+3WtJ2z0hwF2NG1b5kcA==}engines: {node: '>=6.9.0'}@@ -499,8 +650,8 @@ packages:engines: {node: '>=0'}deprecated: This package has been renamed to `fast-tag-pos`- '@dqbd/[email protected]':- resolution: {integrity: sha512-a6I67K1xUkuqcuwulobIJiLikkoE7egMaviI1Jg5bxSn2V7QGqXsGE3jTKr8UIOU/o74mAAd5TkeXFNBtaKF4A==}+ '@dqbd/[email protected]':+ resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==}'@flydotio/[email protected]':resolution: {integrity: sha512-L52UAfrOhmAn3T4TxpeRofQOSO+Kctg+uraB4nLzo4mvvh+4Z7HYxSi7Dnq0Kirz+xx6fDIc4OMNT1EdaORecA==}@@ -1475,6 +1626,216 @@ packages:'@sinonjs/[email protected]':resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==}+ '@smithy/[email protected]':+ resolution: {integrity: sha512-0XuhuHQlEqbNQZp7QxxrFTdVWdwxch4vjxYgfInF91hZFkPxf9QDrdQka0KfxFMPqLNzSw0b95uGTrLliQUavQ==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-Uh0Sz9gdUuz538nvkPiyv1DZRX9+D15EKDtnQP5rYVAzM/dnYk3P8cg73jcxyOitPgT3mE3OVj7ky7sibzHWkw==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-DujtuDA7BGEKExJ05W5OdxCoyekcKT3Rhg1ZGeiUWaz2BJIWXjZmsG/DIP4W48GHno7AQwRsaCb8NcBgH3QZpg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-4FTQGAsuwqTzVMmiRVTn0RR9GrbRfkP0wfu/tXWVHd2LgNpTY0uglQpIScXK4NaEyXbB3JmZt8gfVqO50lP8wg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-3tEbUb8t8an226jKB6V/Q2XU/J53lCwCzULuBPEaF4JjSh+FlCMp7TmogE/Aij5J9DwlsZ4VAD/IRDuQ/0ZtMw==}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-hYNVQOqhFQ6vOpenifFME546f0GfJn2OiQ3M0FDmuUu8V/Uiwy2wej7ZXxFBNqdx0R5DZAqWM1l6VRhGz8oE6A==}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-MLb1f5tbBO2X6K4lMEKJvxeLooyg7guq48C2zKr4qM7F2Gpkz4dc+hdSgu77pCJ76jVqFBjZczHYAs6dp15N+g==}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-tlNQYbfpWXHimHqrvgo14DrMAgUBua/cNoz9fMYcDmYej7MAmUcjav/QKQbFc3NrcPxeJ7QClER4tWZmfwoPng==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-7Qynk6NWtTQhnGTTZwks++nJhQ1O54Mzi7fz4PqZOiYXb4Z1Flpb2yRvdALoggTS8xjtohWUM+RygOtB30YL3Q==}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-twpQ/n+3OWZJ7Z+xu43MJErmhB/WO/mMTnqR6PwWQShvSJ/emx5d1N59LQZk6ZpTAeuRWrc+eHhkzTp9NFjNRQ==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-T4dIdCs1d/+/qMpwhJ1DzOhxCZjZHbHazEPJWdB4GDi2HjIZllVzeBEcdJUN0fomV8DURsgOyrbEUzg3vzTaOg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-wWO3xYmFm6WRW8VsEJ5oU6h7aosFXfszlz3Dj176pTij6o21oZnzkCLzShfmRaaCHDkBXWBdO0c4sQAvLFP6zA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-m1F70cPaMBML4HiTgCw5I+jFNtjgz5z5UdGnUbG37vw6kh4UvizFYjqJGHvicfgKMkDL6mXwyPp5mhZg02g5sg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-Xg2jK9Wc/1g/MBMP/EUn2DLspN8LNt+GMe7cgF+Ty3vl+Zvu+VeZU5nmhveU+H8pxyTsjrAkci8NqY6OuvZnjA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-d7ZuwvYgp1+3682Nx0MD3D/HtkmZd49N3JUndYWQXfRZrYEnCWYc8BHcNmVsPAp9gKvlurdg/mubE6b/rPS9MA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-qRHoah49QJ71eemjuS/WhUXB+mpNtwHRWQr77J/m40ewBVVwvo52kYAmb7iuaECgGTTcYxHS4Wmewfwy++ueew==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-PkOwPNeKdvX/jCpn0A8n9/TyoxjGZB8WVoJmm9YzsnAgggTj4CrjpRHlTQw7dlLZ320n1mY1y+nTRUDViKi/3w==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-ukNUyo6rHmusG64lmkjFeXemwYuKge1BJ8CtpVKmrxQxc6rhUX0vebcptFA9MmrGsnLhwnnqeH83VTU9hwOpjA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-GfGfruksi3nXdFok5RhgtOnWe5f6BndzYfmEXISD+5gAGdayFGpjWu5pIqIweTudMtse20bGbc+7MFZXT1Tb8Q==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-hsjtwpIemmCkm3ZV5fd/T0bPIugW1gJXwZ/hpuVubt2hEUApIoUTrf6qIdh9MAWlw0vjMrA1ztJLAwtNaZogvg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-btYxGVqFUARbUrN6VhL9c3dnSviIwBYD9Rz1jHuN1hgh28Fpv2xjU1HeCeDJX68xctz7r4l1PBnFhGg1WBBPuA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-BtEk3FG7Ks64GAbt+JnKqwuobJNX8VmFLBsKIwWr1D60T426fGrV2L3YS5siOcUhhp6/Y6yhBw1PSPxA5p7qGg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-uEC/kCCFto83bz5ZzapcrgGqHOh/0r69sZ2ZuHlgoD5kYgXJEThCoTuw/y1Ub3cE7aaKdznb+jD9xRPIfIwD7g==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-/+OsJRNtoRbtsX0UpSgWVxFZLsJHo/4sTr+kBg/J78sr7iC+tHeOvOJrS5hCpVQ6sWBbhWLp1UNiuMyZhE6pmA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-fDo3m7YqXBs7neciOePPd/X9LPm5QLlDMdIC4m1H6dgNLnXfLMFNIxEfPyohGA8VW9Wn4X8lygnPSGxDZSmp0Q==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-NsV1jF4EvmO5wqmaSzlnTVetemBS3FZHdyc5CExbDljcyJCEEkJr8ANu2JvtNbVg/9MvKAWV44kTrGS+Pi4INg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-dxw1BDxJiY9/zI3cBqfVrInij6ShjpV4fmGHesGZZUiP9OSE/EVfdwdRz0PgvkEvrZHpsj2htRaHJfftE8giBA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-8VXK/KzOHefoC65yRgCn5vG1cysPJjHnOVt9d0ybFQSmJgQj152vMn4EkYhGuaOmnnZvCPav/KnYyE6/KsNZ2w==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-4FdOhwpTW7jtSFWm7SpfLGKIBC9ZaTKG5nBF0wK24aoQKQyDIKUw3+KFWCQ9maMzrgTJIuOvOnsV2lLGW5XjTg==}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-9m6NXE0ww+ra5HKHCHig20T+FAwxBAm7DIdwc/767uGWbRcY720ybgPacQNB96JMOI7xVr/CDa3oMzKmW4a+kw==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-fRw7zymjIDt6XxIsLwfJfYUfbGoO9CmCJk6rjJ/X5cd20+d2Is7xjU5Kt/AiDt6hX8DAf5dztmfP5O82gR9emA==}+ engines: {node: '>= 10.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-H3BSZdBDiVZGzt8TG51Pd2FvFO0PAx/A0mJ0EH8a13KJ6iUCdYnw/Dk/MdC1kTd0eUuUGisDFaxXVXo4HHFL1g==}+ engines: {node: '>= 10.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-kPt8j4emm7rdMWQyL0F89o92q10gvCUa6sBkBtDJ7nV2+P7wpXczzOfoDJ49CKXe5CCqb8dc1W+ZdLlrKzSAnQ==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-7UtIE9eH0u41zpB60Jzr0oNCQ3hMJUabMcKRUVjmyHTXiWDE4vjSqN6qlih7rCNeKGbioS7f/y2Jgym4QZcKFg==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-6hhckcBqVgjWAqLy2vqlPZ3rfxLDhFWEmM7oLh2POGvsi7j0tHkbN7w4DFhuBExVJAbJ/qqxqZdRY6Fu7/OezQ==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-p7iYAPaQjoeM+AKABpYWeDdtwQNxasr4aXQEA/OmbOaug9V0odRVDy3Wx4ci8soljE/JXQo+abV0qZpW8NX0yA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-TCEhLnY581YJ+g1x0hapPz13JFqzmh/pMWL2KEFASC51qCfw3+Y47MrTmea4bUE5vsdxQ4F6/KFbUeSz22Q1ow==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-R3ufuzJRxSJbE58K9AEnL/uSZyVdHzud9wLS8tIbXclxKzoe09CRohj2xV8wpx5tj7ZbiJaKYcutMm1eYgz/0A==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-/jL/V1xdVRt5XppwiaEU8Etp5WHZj609n0xMTuehmCqdoOFbId1M+aEeDWZsQ+8JbEB/BJ6ynY2SlYmOaKtt8w==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-p/MYV+JmqmPyjdgyN2UxAeYDj9cBqCjp0C/NsTWnnjoZUVqoeZ6IrW915L9CAKWVECgv9lVQGc4u/yz26/bI1A==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==}+ engines: {node: '>=16.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-d5yGlQtmN/z5eoTtIYgkvOw27US2Ous4VycnXatyoImIF9tzlcpnKqQ/V7qhvJmb2p6xZne1NopCLakdTnkBBQ==}+ engines: {node: '>=16.0.0'}+'@stdlib/[email protected]':resolution: {integrity: sha512-3YHwSWiUqGlTLSwxAWxrqaD1PkgcJniGyotJeIt5X0tSNmSW0/c9RWroCImTUUB3zBkyBJ79MyU9Nf4Qgm59fQ==}engines: {node: '>=0.10.0', npm: '>2.7.0'}@@ -2004,6 +2365,9 @@ packages:resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==}+ resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==}+resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==}@@ -2146,6 +2510,9 @@ packages:resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}+ resolution: {integrity: sha512-hSo2/tFV29whjFFtVtdS7kHmtUsjfMO1sgwE/d5bhOE4O7Vkj5G1R9lLIqkIprp/+rrvCq3HGvEaOgry7xRcDA==}+resolution: {integrity: sha512-D116FKTuauCShJjPuOAFnkyAPMhV/6f403+yPZwyyFY6gErK1AA41y9rQdBvj8eHDZ9sXVJ6TzmzObVfAFh3ig==}@@ -2565,6 +2932,10 @@ packages:resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}+ resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==}+ hasBin: true+resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==}@@ -2613,6 +2984,10 @@ packages:resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==}+ resolution: {integrity: sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==}+ engines: {node: '>= 18'}+resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}engines: {node: '>= 6'}@@ -2621,6 +2996,10 @@ packages:resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==}engines: {node: '>= 12.20'}+ resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==}+ engines: {node: '>= 18'}+resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}engines: {node: '>=12.20.0'}@@ -3107,6 +3486,9 @@ packages:resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==}+ resolution: {integrity: sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==}+resolution: {integrity: sha512-ESjNzSlt/sWE8sciZH8kBF8BPlwXPwhR6pWKAw8bw4Bwj+iZcnKW6ONWUutJ7eObuBZQpiIb8S7OYspWrKt7rA==}engines: {node: '>=14'}@@ -4027,6 +4409,10 @@ packages:resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==}engines: {node: '>=0.6'}+ resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==}+ engines: {node: '>=0.6'}+resolution: {integrity: sha512-x+NLUpx9SYrcwXtX7ob1gnkSems4i/mGZX5SlYxwIau6RrUSODO89TR/XDGGpn5RPWSYIB+aSfuSlV5+CmbTBg==}engines: {node: '>=0.6'}@@ -4365,6 +4751,9 @@ packages:resolution: {integrity: sha512-syeEEd112om/waJ5gOQ+SaYi+setuidQ4ZIPiQREF4yJeegXhn2HKy6C0JYm7uhVQKfMAvuZ22dIRsnoDv7AMw==}engines: {node: '>=12.*'}+ resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==}+resolution: {integrity: sha512-h2J6kKEikXnZyurUcCYg215qkQpINOhdWkiclHcWAuVeqXsNrfrYaf1s0qbbcdRyMtrVW48I+VdVTw71Cnn20Q==}engines: {npm: '>=8'}@@ -4526,8 +4915,8 @@ packages:engines: {node: '>=14.17'}hasBin: true- resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}+ resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==}engines: {node: '>=14.17'}hasBin: true@@ -4797,6 +5186,503 @@ snapshots:transitivePeerDependencies:- encoding+ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/util': 3.0.0+ '@aws-sdk/types': 3.679.0+ tslib: 1.14.1++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-crypto/supports-web-crypto': 5.2.0+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-locate-window': 3.679.0+ '@smithy/util-utf8': 2.3.0+ tslib: 2.6.3++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/types': 3.679.0+ tslib: 2.6.3++ '@aws-crypto/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-utf8-browser': 3.259.0+ tslib: 1.14.1++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/util-utf8': 2.3.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])+ '@aws-sdk/client-sts': 3.679.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])+ '@aws-sdk/middleware-host-header': 3.679.0+ '@aws-sdk/middleware-logger': 3.679.0+ '@aws-sdk/middleware-recursion-detection': 3.679.0+ '@aws-sdk/middleware-user-agent': 3.679.0+ '@aws-sdk/region-config-resolver': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-endpoints': 3.679.0+ '@aws-sdk/util-user-agent-browser': 3.679.0+ '@aws-sdk/util-user-agent-node': 3.679.0+ '@smithy/config-resolver': 3.0.10+ '@smithy/core': 2.5.1+ '@smithy/fetch-http-handler': 3.2.9+ '@smithy/hash-node': 3.0.8+ '@smithy/invalid-dependency': 3.0.8+ '@smithy/middleware-content-length': 3.0.10+ '@smithy/middleware-endpoint': 3.2.1+ '@smithy/middleware-retry': 3.0.25+ '@smithy/middleware-serde': 3.0.8+ '@smithy/middleware-stack': 3.0.8+ '@smithy/node-config-provider': 3.1.9+ '@smithy/node-http-handler': 3.2.5+ '@smithy/protocol-http': 4.1.5+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/url-parser': 3.0.8+ '@smithy/util-base64': 3.0.0+ '@smithy/util-body-length-browser': 3.0.0+ '@smithy/util-body-length-node': 3.0.0+ '@smithy/util-defaults-mode-browser': 3.0.25+ '@smithy/util-defaults-mode-node': 3.0.25+ '@smithy/util-endpoints': 2.1.4+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-retry': 3.0.8+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - aws-crt++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])+ '@aws-sdk/client-sts': 3.679.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])+ '@aws-sdk/middleware-host-header': 3.679.0+ '@aws-sdk/middleware-logger': 3.679.0+ '@aws-sdk/middleware-recursion-detection': 3.679.0+ '@aws-sdk/middleware-user-agent': 3.679.0+ '@aws-sdk/region-config-resolver': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-endpoints': 3.679.0+ '@aws-sdk/util-user-agent-browser': 3.679.0+ '@aws-sdk/util-user-agent-node': 3.679.0+ '@smithy/config-resolver': 3.0.10+ '@smithy/core': 2.5.1+ '@smithy/fetch-http-handler': 3.2.9+ '@smithy/hash-node': 3.0.8+ '@smithy/invalid-dependency': 3.0.8+ '@smithy/middleware-content-length': 3.0.10+ '@smithy/middleware-endpoint': 3.2.1+ '@smithy/middleware-retry': 3.0.25+ '@smithy/middleware-serde': 3.0.8+ '@smithy/middleware-stack': 3.0.8+ '@smithy/node-config-provider': 3.1.9+ '@smithy/node-http-handler': 3.2.5+ '@smithy/protocol-http': 4.1.5+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/url-parser': 3.0.8+ '@smithy/util-base64': 3.0.0+ '@smithy/util-body-length-browser': 3.0.0+ '@smithy/util-body-length-node': 3.0.0+ '@smithy/util-defaults-mode-browser': 3.0.25+ '@smithy/util-defaults-mode-node': 3.0.25+ '@smithy/util-endpoints': 2.1.4+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-retry': 3.0.8+ '@smithy/util-utf8': 3.0.0+ '@smithy/util-waiter': 3.1.7+ '@types/uuid': 9.0.8+ tslib: 2.6.3+ uuid: 9.0.1+ transitivePeerDependencies:+ - aws-crt++ '@aws-sdk/[email protected](@aws-sdk/[email protected])':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/client-sts': 3.679.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])+ '@aws-sdk/middleware-host-header': 3.679.0+ '@aws-sdk/middleware-logger': 3.679.0+ '@aws-sdk/middleware-recursion-detection': 3.679.0+ '@aws-sdk/middleware-user-agent': 3.679.0+ '@aws-sdk/region-config-resolver': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-endpoints': 3.679.0+ '@aws-sdk/util-user-agent-browser': 3.679.0+ '@aws-sdk/util-user-agent-node': 3.679.0+ '@smithy/config-resolver': 3.0.10+ '@smithy/core': 2.5.1+ '@smithy/fetch-http-handler': 3.2.9+ '@smithy/hash-node': 3.0.8+ '@smithy/invalid-dependency': 3.0.8+ '@smithy/middleware-content-length': 3.0.10+ '@smithy/middleware-endpoint': 3.2.1+ '@smithy/middleware-retry': 3.0.25+ '@smithy/middleware-serde': 3.0.8+ '@smithy/middleware-stack': 3.0.8+ '@smithy/node-config-provider': 3.1.9+ '@smithy/node-http-handler': 3.2.5+ '@smithy/protocol-http': 4.1.5+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/url-parser': 3.0.8+ '@smithy/util-base64': 3.0.0+ '@smithy/util-body-length-browser': 3.0.0+ '@smithy/util-body-length-node': 3.0.0+ '@smithy/util-defaults-mode-browser': 3.0.25+ '@smithy/util-defaults-mode-node': 3.0.25+ '@smithy/util-endpoints': 2.1.4+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-retry': 3.0.8+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - aws-crt++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/middleware-host-header': 3.679.0+ '@aws-sdk/middleware-logger': 3.679.0+ '@aws-sdk/middleware-recursion-detection': 3.679.0+ '@aws-sdk/middleware-user-agent': 3.679.0+ '@aws-sdk/region-config-resolver': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-endpoints': 3.679.0+ '@aws-sdk/util-user-agent-browser': 3.679.0+ '@aws-sdk/util-user-agent-node': 3.679.0+ '@smithy/config-resolver': 3.0.10+ '@smithy/core': 2.5.1+ '@smithy/fetch-http-handler': 3.2.9+ '@smithy/hash-node': 3.0.8+ '@smithy/invalid-dependency': 3.0.8+ '@smithy/middleware-content-length': 3.0.10+ '@smithy/middleware-endpoint': 3.2.1+ '@smithy/middleware-retry': 3.0.25+ '@smithy/middleware-serde': 3.0.8+ '@smithy/middleware-stack': 3.0.8+ '@smithy/node-config-provider': 3.1.9+ '@smithy/node-http-handler': 3.2.5+ '@smithy/protocol-http': 4.1.5+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/url-parser': 3.0.8+ '@smithy/util-base64': 3.0.0+ '@smithy/util-body-length-browser': 3.0.0+ '@smithy/util-body-length-node': 3.0.0+ '@smithy/util-defaults-mode-browser': 3.0.25+ '@smithy/util-defaults-mode-node': 3.0.25+ '@smithy/util-endpoints': 2.1.4+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-retry': 3.0.8+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - aws-crt++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])+ '@aws-sdk/middleware-host-header': 3.679.0+ '@aws-sdk/middleware-logger': 3.679.0+ '@aws-sdk/middleware-recursion-detection': 3.679.0+ '@aws-sdk/middleware-user-agent': 3.679.0+ '@aws-sdk/region-config-resolver': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-endpoints': 3.679.0+ '@aws-sdk/util-user-agent-browser': 3.679.0+ '@aws-sdk/util-user-agent-node': 3.679.0+ '@smithy/config-resolver': 3.0.10+ '@smithy/core': 2.5.1+ '@smithy/fetch-http-handler': 3.2.9+ '@smithy/hash-node': 3.0.8+ '@smithy/invalid-dependency': 3.0.8+ '@smithy/middleware-content-length': 3.0.10+ '@smithy/middleware-endpoint': 3.2.1+ '@smithy/middleware-retry': 3.0.25+ '@smithy/middleware-serde': 3.0.8+ '@smithy/middleware-stack': 3.0.8+ '@smithy/node-config-provider': 3.1.9+ '@smithy/node-http-handler': 3.2.5+ '@smithy/protocol-http': 4.1.5+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/url-parser': 3.0.8+ '@smithy/util-base64': 3.0.0+ '@smithy/util-body-length-browser': 3.0.0+ '@smithy/util-body-length-node': 3.0.0+ '@smithy/util-defaults-mode-browser': 3.0.25+ '@smithy/util-defaults-mode-node': 3.0.25+ '@smithy/util-endpoints': 2.1.4+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-retry': 3.0.8+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - aws-crt++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/core': 2.5.1+ '@smithy/node-config-provider': 3.1.9+ '@smithy/property-provider': 3.1.8+ '@smithy/protocol-http': 4.1.5+ '@smithy/signature-v4': 4.2.1+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/util-middleware': 3.0.8+ fast-xml-parser: 4.4.1+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/client-cognito-identity': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@smithy/property-provider': 3.1.8+ '@smithy/types': 3.6.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - aws-crt++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@smithy/property-provider': 3.1.8+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@smithy/fetch-http-handler': 3.2.9+ '@smithy/node-http-handler': 3.2.5+ '@smithy/property-provider': 3.1.8+ '@smithy/protocol-http': 4.1.5+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/util-stream': 3.2.1+ tslib: 2.6.3++ '@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])':+ dependencies:+ '@aws-sdk/client-sts': 3.679.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/credential-provider-env': 3.679.0+ '@aws-sdk/credential-provider-http': 3.679.0+ '@aws-sdk/credential-provider-process': 3.679.0+ '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))+ '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/[email protected])+ '@aws-sdk/types': 3.679.0+ '@smithy/credential-provider-imds': 3.2.5+ '@smithy/property-provider': 3.1.8+ '@smithy/shared-ini-file-loader': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - '@aws-sdk/client-sso-oidc'+ - aws-crt++ '@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])':+ dependencies:+ '@aws-sdk/credential-provider-env': 3.679.0+ '@aws-sdk/credential-provider-http': 3.679.0+ '@aws-sdk/credential-provider-ini': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])+ '@aws-sdk/credential-provider-process': 3.679.0+ '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))+ '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/[email protected])+ '@aws-sdk/types': 3.679.0+ '@smithy/credential-provider-imds': 3.2.5+ '@smithy/property-provider': 3.1.8+ '@smithy/shared-ini-file-loader': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - '@aws-sdk/client-sso-oidc'+ - '@aws-sdk/client-sts'+ - aws-crt++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@smithy/property-provider': 3.1.8+ '@smithy/shared-ini-file-loader': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ dependencies:+ '@aws-sdk/client-sso': 3.679.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/token-providers': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))+ '@aws-sdk/types': 3.679.0+ '@smithy/property-provider': 3.1.8+ '@smithy/shared-ini-file-loader': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - '@aws-sdk/client-sso-oidc'+ - aws-crt++ '@aws-sdk/[email protected](@aws-sdk/[email protected])':+ dependencies:+ '@aws-sdk/client-sts': 3.679.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@smithy/property-provider': 3.1.8+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ dependencies:+ '@aws-sdk/client-cognito-identity': 3.679.0+ '@aws-sdk/client-sso': 3.679.0+ '@aws-sdk/client-sts': 3.679.0+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/credential-provider-cognito-identity': 3.679.0+ '@aws-sdk/credential-provider-env': 3.679.0+ '@aws-sdk/credential-provider-http': 3.679.0+ '@aws-sdk/credential-provider-ini': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])+ '@aws-sdk/credential-provider-process': 3.679.0+ '@aws-sdk/credential-provider-sso': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))+ '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/[email protected])+ '@aws-sdk/types': 3.679.0+ '@smithy/credential-provider-imds': 3.2.5+ '@smithy/property-provider': 3.1.8+ '@smithy/types': 3.6.0+ tslib: 2.6.3+ transitivePeerDependencies:+ - '@aws-sdk/client-sso-oidc'+ - aws-crt++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/protocol-http': 4.1.5+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/protocol-http': 4.1.5+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@aws-sdk/util-endpoints': 3.679.0+ '@smithy/core': 2.5.1+ '@smithy/protocol-http': 4.1.5+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@smithy/protocol-http': 1.2.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/node-config-provider': 3.1.9+ '@smithy/types': 3.6.0+ '@smithy/util-config-provider': 3.0.0+ '@smithy/util-middleware': 3.0.8+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@smithy/signature-v4': 1.1.0+ tslib: 2.6.3++ dependencies:+ '@aws-sdk/client-sso-oidc': 3.679.0(@aws-sdk/[email protected])+ '@aws-sdk/types': 3.679.0+ '@smithy/property-provider': 3.1.8+ '@smithy/shared-ini-file-loader': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/types': 3.6.0+ '@smithy/util-endpoints': 2.1.4+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.679.0+ '@smithy/types': 3.6.0+ bowser: 2.11.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/middleware-user-agent': 3.679.0+ '@aws-sdk/types': 3.679.0+ '@smithy/node-config-provider': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@aws-sdk/[email protected]':+ dependencies:+ tslib: 2.6.3+'@babel/[email protected]':dependencies:'@babel/highlight': 7.24.6@@ -5040,7 +5926,7 @@ snapshots:'@devil7softwares/[email protected]': {}- '@dqbd/[email protected]': {}+ '@dqbd/[email protected]': {}'@flydotio/[email protected]':dependencies:@@ -5338,13 +6224,13 @@ snapshots:'@js-sdsl/[email protected]': {}- '@langchain/[email protected]([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))':+ '@langchain/[email protected]([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))':dependencies:ansi-styles: 5.2.0camelcase: 6.3.0decamelize: 1.2.0js-tiktoken: 1.0.12- langsmith: 0.1.34(@langchain/[email protected]([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])))([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))+ langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu)ml-distance: 4.0.1mustache: 4.2.0p-queue: 6.6.2@@ -5356,9 +6242,9 @@ snapshots:- langchain- openai- '@langchain/[email protected]([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))':+ '@langchain/[email protected]([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))':dependencies:- '@langchain/core': 0.2.12([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))js-tiktoken: 1.0.12openai: 4.57.0([email protected])zod: 3.23.8@@ -5367,9 +6253,9 @@ snapshots:- encoding- langchain- '@langchain/[email protected]([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))':+ '@langchain/[email protected]([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))':dependencies:- '@langchain/core': 0.2.12([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))js-tiktoken: 1.0.12transitivePeerDependencies:- langchain@@ -6495,6 +7381,340 @@ snapshots:dependencies:'@sinonjs/commons': 3.0.1+ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/node-config-provider': 3.1.9+ '@smithy/types': 3.6.0+ '@smithy/util-config-provider': 3.0.0+ '@smithy/util-middleware': 3.0.8+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/middleware-serde': 3.0.8+ '@smithy/protocol-http': 4.1.5+ '@smithy/types': 3.6.0+ '@smithy/util-body-length-browser': 3.0.0+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-stream': 3.2.1+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/node-config-provider': 3.1.9+ '@smithy/property-provider': 3.1.8+ '@smithy/types': 3.6.0+ '@smithy/url-parser': 3.0.8+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@aws-crypto/crc32': 3.0.0+ '@smithy/types': 1.2.0+ '@smithy/util-hex-encoding': 1.1.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/protocol-http': 4.1.5+ '@smithy/querystring-builder': 3.0.8+ '@smithy/types': 3.6.0+ '@smithy/util-base64': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/protocol-http': 4.1.5+ '@smithy/querystring-builder': 3.0.8+ '@smithy/types': 3.6.0+ '@smithy/util-base64': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ '@smithy/util-buffer-from': 3.0.0+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/protocol-http': 4.1.5+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/core': 2.5.1+ '@smithy/middleware-serde': 3.0.8+ '@smithy/node-config-provider': 3.1.9+ '@smithy/shared-ini-file-loader': 3.1.9+ '@smithy/types': 3.6.0+ '@smithy/url-parser': 3.0.8+ '@smithy/util-middleware': 3.0.8+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/node-config-provider': 3.1.9+ '@smithy/protocol-http': 4.1.5+ '@smithy/service-error-classification': 3.0.8+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-retry': 3.0.8+ tslib: 2.6.3+ uuid: 9.0.1++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/property-provider': 3.1.8+ '@smithy/shared-ini-file-loader': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/abort-controller': 3.1.6+ '@smithy/protocol-http': 4.1.5+ '@smithy/querystring-builder': 3.0.8+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 1.2.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ '@smithy/util-uri-escape': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/eventstream-codec': 1.1.0+ '@smithy/is-array-buffer': 1.1.0+ '@smithy/types': 1.2.0+ '@smithy/util-hex-encoding': 1.1.0+ '@smithy/util-middleware': 1.1.0+ '@smithy/util-uri-escape': 1.1.0+ '@smithy/util-utf8': 1.1.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/is-array-buffer': 3.0.0+ '@smithy/protocol-http': 4.1.5+ '@smithy/types': 3.6.0+ '@smithy/util-hex-encoding': 3.0.0+ '@smithy/util-middleware': 3.0.8+ '@smithy/util-uri-escape': 3.0.0+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/core': 2.5.1+ '@smithy/middleware-endpoint': 3.2.1+ '@smithy/middleware-stack': 3.0.8+ '@smithy/protocol-http': 4.1.5+ '@smithy/types': 3.6.0+ '@smithy/util-stream': 3.2.1+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/querystring-parser': 3.0.8+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/util-buffer-from': 3.0.0+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/is-array-buffer': 1.1.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/is-array-buffer': 2.2.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/is-array-buffer': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/property-provider': 3.1.8+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ bowser: 2.11.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/config-resolver': 3.0.10+ '@smithy/credential-provider-imds': 3.2.5+ '@smithy/node-config-provider': 3.1.9+ '@smithy/property-provider': 3.1.8+ '@smithy/smithy-client': 3.4.2+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/node-config-provider': 3.1.9+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/service-error-classification': 3.0.8+ '@smithy/types': 3.6.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/fetch-http-handler': 4.0.0+ '@smithy/node-http-handler': 3.2.5+ '@smithy/types': 3.6.0+ '@smithy/util-base64': 3.0.0+ '@smithy/util-buffer-from': 3.0.0+ '@smithy/util-hex-encoding': 3.0.0+ '@smithy/util-utf8': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/util-buffer-from': 1.1.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/util-buffer-from': 2.2.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/util-buffer-from': 3.0.0+ tslib: 2.6.3++ '@smithy/[email protected]':+ dependencies:+ '@smithy/abort-controller': 3.1.6+ '@smithy/types': 3.6.0+ tslib: 2.6.3+'@stdlib/[email protected]': {}'@stdlib/[email protected]':@@ -7106,6 +8326,8 @@ snapshots:+ [email protected]: {}+dependencies:balanced-match: 1.0.2@@ -7263,6 +8485,25 @@ snapshots:+ dependencies:+ '@aws-sdk/client-sagemaker': 3.679.0+ '@aws-sdk/credential-providers': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))+ '@aws-sdk/protocol-http': 3.374.0+ '@aws-sdk/signature-v4': 3.374.0+ form-data: 4.0.0+ form-data-encoder: 4.0.2+ formdata-node: 6.0.3+ js-base64: 3.7.2+ node-fetch: 2.7.0+ qs: 6.11.2+ readable-stream: 4.5.2+ url-join: 4.0.1+ transitivePeerDependencies:+ - '@aws-sdk/client-sso-oidc'+ - aws-crt+ - encoding+@@ -7650,6 +8891,10 @@ snapshots:+ dependencies:+ strnum: 1.0.5+dependencies:bser: 2.1.1@@ -7701,6 +8946,8 @@ snapshots:+ [email protected]: {}+dependencies:asynckit: 0.4.0@@ -7712,6 +8959,8 @@ snapshots:node-domexception: 1.0.0web-streams-polyfill: 4.0.0-beta.3+ [email protected]: {}+dependencies:fetch-blob: 3.2.0@@ -8447,6 +9696,8 @@ snapshots:+ [email protected]: {}+dependencies:config-chain: 1.1.13@@ -8513,17 +9764,17 @@ snapshots:- [email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]):+ [email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]):dependencies:- '@langchain/core': 0.2.12([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))- '@langchain/openai': 0.2.1([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))- '@langchain/textsplitters': 0.0.3([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))+ '@langchain/openai': 0.2.1([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))+ '@langchain/textsplitters': 0.0.3([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))binary-extensions: 2.3.0js-tiktoken: 1.0.12js-yaml: 4.1.0jsonpointer: 5.0.1langchainhub: 0.0.11- langsmith: 0.1.34(@langchain/[email protected]([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])))([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))+ langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu)ml-distance: 4.0.1openapi-types: 12.1.3p-retry: 4.6.2@@ -8532,14 +9783,16 @@ snapshots:zod: 3.23.8zod-to-json-schema: 3.23.1([email protected])optionalDependencies:+ '@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])'@supabase/supabase-js': 2.44.2axios: 1.7.2cheerio: 1.0.0-rc.12+ fast-xml-parser: 4.4.1handlebars: 4.7.8html-to-text: 9.0.5ioredis: 5.4.1mammoth: 1.7.2- mongodb: 6.6.2([email protected])+ mongodb: 6.6.2(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])pdf-parse: 1.1.1puppeteer: 22.12.1([email protected])redis: 4.6.14@@ -8550,7 +9803,7 @@ snapshots:- [email protected](@langchain/[email protected]([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])))([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected])):+ [email protected](zyeavx4tfqw3smbbpiinhfxxeu):dependencies:'@types/uuid': 9.0.8commander: 10.0.1@@ -8559,8 +9812,8 @@ snapshots:p-retry: 4.6.2uuid: 9.0.1optionalDependencies:- '@langchain/core': 0.2.12([email protected](@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))- langchain: 0.2.8(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])+ '@langchain/core': 0.2.12([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))+ langchain: 0.2.8(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected]))(@supabase/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])openai: 4.57.0([email protected])@@ -8768,19 +10021,20 @@ snapshots:'@types/whatwg-url': 11.0.5whatwg-url: 13.0.0+ [email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]):dependencies:'@mongodb-js/saslprep': 1.1.7bson: 6.8.0mongodb-connection-string-url: 3.0.1optionalDependencies:+ '@aws-sdk/credential-providers': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))socks: 2.8.3+ [email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]):dependencies:bson: 6.8.0kareem: 2.6.3- mongodb: 6.6.2([email protected])+ mongodb: 6.6.2(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])mpath: 0.9.0mquery: 5.0.0ms: 2.1.3@@ -8829,7 +10083,7 @@ snapshots:+ [email protected](@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected]):dependencies:afinn-165: 1.0.4afinn-165-financialmarketnews: 3.0.0@@ -8837,7 +10091,7 @@ snapshots:dotenv: 16.4.5http-server: 14.1.1memjs: 1.3.2- mongoose: 8.4.4([email protected])+ mongoose: 8.4.4(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])pg: 8.12.0redis: 4.6.14safe-stable-stringify: 2.4.3@@ -9208,7 +10462,7 @@ snapshots:csv-parse: 5.5.6gpt3-tokenizer: 1.1.5openai: 3.3.0- typescript: 5.6.2+ typescript: 5.6.3uuid: 9.0.1zod: 3.23.8transitivePeerDependencies:@@ -9295,6 +10549,10 @@ snapshots:dependencies:side-channel: 1.0.6+ dependencies:+ side-channel: 1.0.6+dependencies:side-channel: 1.0.6@@ -9643,6 +10901,8 @@ snapshots:'@types/node': 20.14.1qs: 6.12.2+ [email protected]: {}+dependencies:bin-links: 4.0.4@@ -9806,7 +11066,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}[email protected](@babel/[email protected]):dependencies:diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsnew file mode 100644index 000000000..f4e93e7fc--- /dev/null+++ b/apps/api/src/controllers/v1/extract.ts@@ -0,0 +1,197 @@+import { Request, Response } from "express";+import { Logger } from "../../lib/logger";+import {+ Document,+ legacyDocumentConverter,+ legacyExtractorOptions,+ legacyScrapeOptions,+ RequestWithAuth,+ ExtractRequest,+ extractRequestSchema,+ ExtractResponse,+ legacyCrawlerOptions,+ MapDocument,+} from "./types";+import { billTeam } from "../../services/billing/credit_billing";+import { v4 as uuidv4 } from "uuid";+import { numTokensFromString } from "../../lib/LLM-extraction/helpers";+import { addScrapeJob, waitForJob } from "../../services/queue-jobs";+import { logJob } from "../../services/logging/log_job";+import { getJobPriority } from "../../lib/job-priority";+import { PlanType } from "../../types";+import { getMapResults } from "./map";+import { rerankDocuments } from "../../lib/extract/reranker";+import { generateBasicCompletion } from "../../lib/extract/completions";++++export async function extractController(+ req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,+ res: Response<ExtractResponse>+) {+ req.body = extractRequestSchema.parse(req.body);+ let earlyReturn = false;++ const origin = req.body.origin;+ const timeout = req.body.timeout;+// const pageOptions = legacyScrapeOptions(req.body);+// const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;+ const jobId = uuidv4();++ const startTime = new Date().getTime();+ const jobPriority = await getJobPriority({+ plan: req.auth.plan as PlanType,+ team_id: req.auth.team_id,+ basePriority: 10,+ });++ const urls = req.body.urls;+ const mappedDocuments: MapDocument[] = [];++ const prompt = req.body.prompt;+ const keywords = await generateBasicCompletion(`If the user's prompt is: "${prompt}", what are the most important keywords besides the extraction task? Output only the keywords, separated by commas.`);++ for (const url of urls) {+ if (url.endsWith("/*")) {+ const mapResults = await getMapResults({+ url: url.slice(0, -2),+ search: req.body.prompt,+ limit: 100,+ ignoreSitemap: true,+ includeSubdomains: false,+ crawlerOptions: {},+ teamId: req.auth.team_id,+ plan: req.auth.plan,+ origin: req.body.origin,+ subId: req.acuc?.sub_id,+ includeMetadata: true+ });+ // top 3 links+ const top3Links = (mapResults.links as MapDocument[]).slice(0, 3);+ console.log(top3Links);+ // console.log(top3Links);+ mappedDocuments.push(...(mapResults.links as MapDocument[]));+ // transform mappedUrls to just documents+ // we quickly rerank+ const rerank = await rerankDocuments(mappedDocuments.map(x => `URL: ${x.url}\nTITLE: ${x.title}\nDESCRIPTION: ${x.description}`), "What URLs are most relevant to the following prompt: " + req.body.prompt.toLocaleLowerCase().replace("extract", " ").replace("extract ", " "));+ console.log(rerank);+ } else {+ mappedDocuments.push({ url });+ }+ }++ req.body.urls = mappedDocuments.map(x => x.url);++++// const job = await addScrapeJob(+// {+// url: req.body.url,+// mode: "single_urls",+// crawlerOptions: {},+// team_id: req.auth.team_id,+// plan: req.auth.plan,+// pageOptions,+// extractorOptions,+// origin: req.body.origin,+// is_scrape: true,+// },+// {},+// jobId,+// jobPriority+// );++// const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds : 0) + a, 0);++// let doc: any | undefined;+// try {+// doc = (await waitForJob(job.id, timeout + totalWait))[0];+// } catch (e) {+// Logger.error(`Error in scrapeController: ${e}`);+// if (e instanceof Error && e.message.startsWith("Job wait")) {+// return res.status(408).json({+// success: false,+// error: "Request timed out",+// });+// } else {+// return res.status(500).json({+// success: false,+// error: `(Internal server error) - ${e && e?.message ? e.message : e} ${+// extractorOptions && extractorOptions.mode !== "markdown"+// ? " - Could be due to LLM parsing issues"+// : ""+// }`,+// });+// }+// }++// await job.remove();++// if (!doc) {+// console.error("!!! PANIC DOC IS", doc, job);+// return res.status(200).json({+// success: true,+// warning: "No page found",+// data: doc,+// });+// }++// delete doc.index;+// delete doc.provider;++// const endTime = new Date().getTime();+// const timeTakenInSeconds = (endTime - startTime) / 1000;+// const numTokens =+// doc && doc.markdown+// ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")+// : 0;++// let creditsToBeBilled = 1; // Assuming 1 credit per document+// if (earlyReturn) {+// // Don't bill if we're early returning+// return;+// }+// if(req.body.extract && req.body.formats.includes("extract")) {+// creditsToBeBilled = 5;+// }++// billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(error => {+// Logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`);+// // Optionally, you could notify an admin or add to a retry queue here+// });++// if (!pageOptions || !pageOptions.includeRawHtml) {+// if (doc && doc.rawHtml) {+// delete doc.rawHtml;+// }+// }++// if(pageOptions && pageOptions.includeExtract) {+// if(!pageOptions.includeMarkdown && doc && doc.markdown) {+// delete doc.markdown;+// }+// }++// logJob({+// job_id: jobId,+// success: true,+// message: "Scrape completed",+// num_docs: 1,+// docs: [doc],+// time_taken: timeTakenInSeconds,+// team_id: req.auth.team_id,+// mode: "scrape",+// url: req.body.url,+// crawlerOptions: {},+// pageOptions: pageOptions,+// origin: origin,+// extractor_options: extractorOptions,+// num_tokens: numTokens,+// });++ return res.status(200).json({+ success: true,+ data: null,+ scrape_id: origin?.includes("website") ? jobId : undefined,+ });+}diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 5ed3dd51c..7e74b43e4 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -15,11 +15,11 @@ import {removeDuplicateUrls,} from "../../lib/validateUrl";import { fireEngineMap } from "../../search/fireEngine";-import { billTeam } from "../../services/billing/credit_billing";-import { logJob } from "../../services/logging/log_job";import { performCosineSimilarity } from "../../lib/map-cosine";import { Logger } from "../../lib/logger";import Redis from "ioredis";+import { billTeam } from "../../services/billing/credit_billing";+import { logJob } from "../../services/logging/log_job";configDotenv();const redis = new Redis(process.env.REDIS_URL);@@ -29,35 +29,50 @@ const MAX_MAP_LIMIT = 5000;// Max Links that "Smart /map" can returnconst MAX_FIRE_ENGINE_RESULTS = 1000;-export async function mapController(- req: RequestWithAuth<{}, MapResponse, MapRequest>,- res: Response<MapResponse>-) {- const startTime = new Date().getTime();-- req.body = mapRequestSchema.parse(req.body);-- const limit: number = req.body.limit ?? MAX_MAP_LIMIT;+interface MapOptions {+ url: string;+ search?: string;+ limit?: number;+ ignoreSitemap?: boolean;+ includeSubdomains?: boolean;+ crawlerOptions?: any;+ teamId: string;+ plan: string;+ origin?: string;+ subId?: string;+ includeMetadata?: boolean;+}+export async function getMapResults({+ url,+ search,+ limit = MAX_MAP_LIMIT,+ ignoreSitemap = false,+ includeSubdomains = false,+ crawlerOptions = {},+ teamId,+ plan,+ origin,+ subId,+ includeMetadata = false,+}: MapOptions) {+ const startTime = new Date().getTime();const id = uuidv4();- let links: string[] = [req.body.url];+ let links: { url: string; title?: string; description?: string }[] = [{ url }];const sc: StoredCrawl = {- originUrl: req.body.url,- crawlerOptions: legacyCrawlerOptions(req.body),+ originUrl: url,+ crawlerOptions,pageOptions: {},- team_id: req.auth.team_id,+ team_id: teamId,createdAt: Date.now(),- plan: req.auth.plan,+ plan,};const crawler = crawlToCrawler(id, sc);- let urlWithoutWww = req.body.url.replace("www.", "");-- let mapUrl = req.body.search- ? `"${req.body.search}" site:${urlWithoutWww}`- : `site:${req.body.url}`;+ let urlWithoutWww = url.replace("www.", "");+ let mapUrl = search ? `"${search}" site:${urlWithoutWww}` : `site:${url}`;const resultsPerPage = 100;const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);@@ -81,12 +96,11 @@ export async function mapController(pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1));allResults = await Promise.all(pagePromises);- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours+ await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60);}- // Parallelize sitemap fetch with serper searchconst [sitemap, ...searchResults] = await Promise.all([- req.body.ignoreSitemap ? null : crawler.tryGetSitemap(),+ ignoreSitemap ? null : crawler.tryGetSitemap(),...(cachedResult ? [] : pagePromises),]);@@ -96,7 +110,7 @@ export async function mapController(if (sitemap !== null) {sitemap.forEach((x) => {- links.push(x.url);+ links.push({ url: x.url });});}@@ -110,67 +124,96 @@ export async function mapController(}if (mapResults.length > 0) {- if (req.body.search) {- // Ensure all map results are first, maintaining their order+ if (search) {links = [- mapResults[0].url,- ...mapResults.slice(1).map((x) => x.url),+ { url: mapResults[0].url, title: mapResults[0].title, description: mapResults[0].description },+ ...mapResults.slice(1).map((x) => ({+ url: x.url,+ title: x.title,+ description: x.description+ })),...links,];} else {- mapResults.map((x) => {- links.push(x.url);+ mapResults.forEach((x) => {+ links.push({+ url: x.url,+ title: x.title,+ description: x.description+ });});}}- // Perform cosine similarity between the search query and the list of links- if (req.body.search) {- const searchQuery = req.body.search.toLowerCase();-- links = performCosineSimilarity(links, searchQuery);+ if (search) {+ const filteredLinks = performCosineSimilarity(links.map(l => l.url), search.toLowerCase());+ links = links.filter(l => filteredLinks.includes(l.url));}links = links.map((x) => {try {- return checkAndUpdateURLForMap(x).url.trim();+ return { ...x, url: checkAndUpdateURLForMap(x.url).url.trim() };} catch (_) {return null;}}).filter((x) => x !== null);- // allows for subdomains to be included- links = links.filter((x) => isSameDomain(x, req.body.url));+ links = links.filter((x) => isSameDomain(x.url, url));- // if includeSubdomains is false, filter out subdomains- if (!req.body.includeSubdomains) {- links = links.filter((x) => isSameSubdomain(x, req.body.url));+ if (!includeSubdomains) {+ links = links.filter((x) => isSameSubdomain(x.url, url));}- // remove duplicates that could be due to http/https or www- links = removeDuplicateUrls(links);-- billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {- Logger.error(- `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`- );- // Optionally, you could notify an admin or add to a retry queue here- });+ links = removeDuplicateUrls(links.map(l => l.url)).map(url => links.find(l => l.url === url));const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - startTime) / 1000;const linksToReturn = links.slice(0, limit);- logJob({- job_id: id,- success: links.length > 0,+ return {+ links: includeMetadata ? linksToReturn : linksToReturn.map(l => l.url),+ scrapeId: origin?.includes("website") ? id : undefined,+ timeTakenInSeconds,+ id,+ linksLength: links.length,+ linksToReturnLength: linksToReturn.length,+ docs: linksToReturn.map(l => l.url),+ };+}++export async function mapController(+ req: RequestWithAuth<{}, MapResponse, MapRequest>,+ res: Response<MapResponse>+) {+ req.body = mapRequestSchema.parse(req.body);++ const results = await getMapResults({+ url: req.body.url,+ search: req.body.search,+ limit: req.body.limit,+ ignoreSitemap: req.body.ignoreSitemap,+ includeSubdomains: req.body.includeSubdomains,+ crawlerOptions: legacyCrawlerOptions(req.body),+ teamId: req.auth.team_id,+ plan: req.auth.plan,+ origin: req.body.origin,+ subId: req.acuc?.sub_id,+ });++ await billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {+ Logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`);+ });++ await logJob({+ job_id: results.id,+ success: results.linksLength > 0,message: "Map completed",- num_docs: linksToReturn.length,- docs: linksToReturn,- time_taken: timeTakenInSeconds,+ num_docs: results.linksToReturnLength,+ docs: results.docs,+ time_taken: results.timeTakenInSeconds,team_id: req.auth.team_id,mode: "map",url: req.body.url,@@ -183,55 +226,7 @@ export async function mapController(return res.status(200).json({success: true,- links: linksToReturn,- scrape_id: req.body.origin?.includes("website") ? id : undefined,+ links: results.links.map(l => l.url),+ scrape_id: results.scrapeId,});}--// Subdomain sitemap url checking--// // For each result, check for subdomains, get their sitemaps and add them to the links-// const processedUrls = new Set();-// const processedSubdomains = new Set();--// for (const result of links) {-// let url;-// let hostParts;-// try {-// url = new URL(result);-// hostParts = url.hostname.split('.');-// } catch (e) {-// continue;-// }--// console.log("hostParts", hostParts);-// // Check if it's a subdomain (more than 2 parts, and not 'www')-// if (hostParts.length > 2 && hostParts[0] !== 'www') {-// const subdomain = hostParts[0];-// console.log("subdomain", subdomain);-// const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`;-// console.log("subdomainUrl", subdomainUrl);--// if (!processedSubdomains.has(subdomainUrl)) {-// processedSubdomains.add(subdomainUrl);--// const subdomainCrawl = crawlToCrawler(id, {-// originUrl: subdomainUrl,-// crawlerOptions: legacyCrawlerOptions(req.body),-// pageOptions: {},-// team_id: req.auth.team_id,-// createdAt: Date.now(),-// plan: req.auth.plan,-// });-// const subdomainSitemap = await subdomainCrawl.tryGetSitemap();-// if (subdomainSitemap) {-// subdomainSitemap.forEach((x) => {-// if (!processedUrls.has(x.url)) {-// processedUrls.add(x.url);-// links.push(x.url);-// }-// });-// }-// }-// }-// }diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 22ac62947..4e31dd86f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -121,8 +121,21 @@ export const scrapeOptions = z.object({}).strict(strictMessage)+export type ScrapeOptions = z.infer<typeof scrapeOptions>;+export const extractV1Options = z.object({+ urls: url.array(),+ prompt: z.string().optional(),+ schema: z.any().optional(),+ origin: z.string().optional().default("api"),+ timeout: z.number().int().positive().finite().safe().default(60000),+}).strict(strictMessage)++export type ExtractV1Options = z.infer<typeof extractV1Options>;+export const extractRequestSchema = extractV1Options;+export type ExtractRequest = z.infer<typeof extractRequestSchema>;+export const scrapeRequestSchema = scrapeOptions.extend({url,origin: z.string().optional().default("api"),@@ -142,6 +155,8 @@ export const scrapeRequestSchema = scrapeOptions.extend({return obj;});++export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;export const batchScrapeRequestSchema = scrapeOptions.extend({@@ -296,6 +311,21 @@ export interface ScrapeResponseRequestTest {error?: string;}+export type ExtractResponse =+ | ErrorResponse+ | {+ success: true;+ warning?: string;+ data: Document;+ scrape_id?: string;+ };++export interface ExtractResponseRequestTest {+ statusCode: number;+ body: ExtractResponse;+ error?: string;+}+export type CrawlResponse =| ErrorResponse| {@@ -492,3 +522,11 @@ export function legacyDocumentConverter(doc: any): Document {},};}++++export interface MapDocument {+ url: string;+ title?: string;+ description?: string;+}\ No newline at end of filediff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.tsnew file mode 100644index 000000000..230584b4a--- /dev/null+++ b/apps/api/src/lib/extract/completions.ts@@ -0,0 +1,119 @@+import OpenAI from "openai";+import { encoding_for_model } from "@dqbd/tiktoken";+import { TiktokenModel } from "@dqbd/tiktoken";+import { ExtractOptions } from "../../controllers/v1/types";+import { Document } from "../entities";+import { z } from "zod";++const maxTokens = 32000;+const modifier = 4;++export class LLMRefusalError extends Error {+ constructor(refusal: string) {+ super("LLM refused to extract the website's content");+ this.name = "LLMRefusalError";+ }+}++interface GenerateCompletionsParams {+ systemPrompt?: string;+ prompt?: string;+ schema?: any;+ pagesContent: string;+}++export async function generateBasicCompletion(prompt: string) {+ const openai = new OpenAI();+ const model: TiktokenModel =+ (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";++ const completion = await openai.chat.completions.create({+ model,+ messages: [{ role: "user", content: prompt }],+ });++ return completion.choices[0].message.content;+}++export async function generateFinalExtraction({+ pagesContent,+ systemPrompt,+ prompt,+ schema,+}: GenerateCompletionsParams): Promise<{+ content: string;+ metadata: { numTokens: number; warning: string };+}> {+ const openai = new OpenAI();+ const model: TiktokenModel =+ (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";++ let extractionContent = pagesContent;+ let numTokens = 0;+ let warning = "";++ const encoder = encoding_for_model(model);+ try {+ const tokens = encoder.encode(extractionContent);+ numTokens = tokens.length;+ } catch (error) {+ extractionContent = extractionContent.slice(0, maxTokens * modifier);+ warning = `Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (${maxTokens}) we support.`;+ } finally {+ encoder.free();+ }++ if (numTokens > maxTokens) {+ extractionContent = extractionContent.slice(0, maxTokens * modifier);+ warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;+ }++ if (schema && (schema.type === "array" || schema._type === "ZodArray")) {+ schema = {+ type: "object",+ properties: {+ items: schema,+ },+ required: ["items"],+ additionalProperties: false,+ };+ }++ const jsonCompletion = await openai.beta.chat.completions.parse({+ model,+ messages: [+ { role: "system", content: systemPrompt },+ { role: "user", content: [{ type: "text", text: extractionContent }] },+ {+ role: "user",+ content: prompt+ ? `Transform the above content into structured JSON output based on the following user request: ${prompt}`+ : "Transform the above content into structured JSON output.",+ },+ ],+ response_format: schema+ ? {+ type: "json_schema",+ json_schema: {+ name: "websiteContent",+ schema: schema.shape,+ strict: true,+ },+ }+ : { type: "json_object" },+ });++ if (jsonCompletion.choices[0].message.refusal !== null) {+ throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal);+ }++ const extraction = jsonCompletion.choices[0].message.parsed;++ return {+ content: extraction,+ metadata: {+ numTokens,+ warning,+ },+ };+}diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.tsnew file mode 100644index 000000000..30aca441a--- /dev/null+++ b/apps/api/src/lib/extract/reranker.ts@@ -0,0 +1,22 @@+import { CohereClient } from "cohere-ai";+import { MapDocument } from "../../controllers/v1/types";+const cohere = new CohereClient({+ token: process.env.COHERE_API_KEY,+});++export async function rerankDocuments(+ documents: (string | Record<string, string>)[],+ query: string,+ topN = 3,+ model = "rerank-english-v3.0"+) {+ const rerank = await cohere.v2.rerank({+ documents,+ query,+ topN,+ model,+ returnDocuments: true,+ });++ return rerank.results.sort((a, b) => b.relevanceScore - a.relevanceScore).map(x => ({ document: x.document, index: x.index, relevanceScore: x.relevanceScore }));+}