Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(summary): revert to gpt-4o-mini but trim tokens correctly
Gergő Móriczfb590d38e057
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 3cf01b00a..95c0f22e2 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -112,7 +112,6 @@"langfuse": "^3.38.4","langfuse-vercel": "^3.38.4","lodash": "^4.17.21",- "mammoth": "^1.7.2","marked": "^14.1.2","ollama-ai-provider": "^1.2.0","openai": "^5.20.2",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 6b4dc99ac..c10b5de23 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -165,9 +165,6 @@ importers:lodash:specifier: ^4.17.21version: 4.17.21- mammoth:- specifier: ^1.7.2- version: 1.7.2marked:specifier: ^14.1.2version: 14.1.2@@ -3431,10 +3428,6 @@ packages:'@walletconnect/[email protected]':resolution: {integrity: sha512-9koTqyGrM2cqFRW517BPY/iEtUDx2r1+Pwwu5m7sJ7ka79wi3EyqhqcICk/yDmv6jAS1rjKgTKXlEhanYjijcA==}- '@xmldom/[email protected]':- resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==}- engines: {node: '>=10.0.0'}-resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}@@ -3656,9 +3649,6 @@ packages:resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}- resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==}-resolution: {integrity: sha512-v2YAxEmKaBLahNwE1mjp4WON6huMNeuDvagFZW+ASCuA/ku0bXR9hSMw0XpiqMoA3+rmnyck/tPRSFQkoC9Cuw==}@@ -4113,9 +4103,6 @@ packages:resolution: {integrity: sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA==}- resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==}-resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}@@ -4133,9 +4120,6 @@ packages:resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}engines: {node: '>=12'}- resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==}-resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}engines: {node: '>= 0.4'}@@ -4690,9 +4674,6 @@ packages:resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}- resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}-resolution: {integrity: sha512-5tCuY9BV8ujfOpwtAGgsTx9CGUapcFMEEyByLv1B+v2+6DhAcw+Zr0nhQT7uwaZ7DiourxFEscghOR8e1aPLQw==}@@ -5087,9 +5068,6 @@ packages:engines: {node: ^18.0.0 || >=20.0.0}hasBin: true- resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==}-resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==}@@ -5145,9 +5123,6 @@ packages:resolution: {integrity: sha512-4lSWmly2Nsj3LaTxxtFmJWuP3Kx+0hYHEd+aNrcXEWT0nKWaPd9/QZPiMkkC680zeALFGHQdQWjBvnilL+vgWA==}- resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==}-resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==}engines: {node: '>=14'}@@ -5234,9 +5209,6 @@ packages:resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}hasBin: true- resolution: {integrity: sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==}-resolution: {integrity: sha512-CQl19J/g+Hbjbv4Y3mFNNXFEL/5t/KCg8POCuUqd4rMKjGG+j1ybER83hxV58zL+dFI1PTkt3GNFSHRt+d8qEQ==}engines: {node: 14 || >=16.14}@@ -5261,11 +5233,6 @@ packages:resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==}- resolution: {integrity: sha512-MqWU2hcLf1I5QMKyAbfJCvrLxnv5WztrAQyorfZ+WPq7Hk82vZFmvfR2/64ajIPpM4jlq0TXp1xZvp/FFaL1Ug==}- engines: {node: '>=12.0.0'}- hasBin: true-resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}@@ -5557,9 +5524,6 @@ packages:zod:optional: true- resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}-resolution: {integrity: sha512-17Gk/eFsFRAZ80p5eKqv89a57uXjd3NgIf1CaXojATPBuujVc/fQSVhBeAU9JCRB+k7J50WQAyWTxK19T9GgbA==}peerDependencies:@@ -5624,9 +5588,6 @@ packages:resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==}- resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}-resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}@@ -6113,9 +6074,6 @@ packages:resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==}engines: {node: '>= 0.4'}- resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==}-resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}@@ -6542,9 +6500,6 @@ packages:resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==}- resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}-resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}@@ -6890,10 +6845,6 @@ packages:resolution: {integrity: sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==}engines: {node: '>=4.0.0'}- resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==}- engines: {node: '>=4.0'}-resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==}engines: {node: '>=4.0'}@@ -11435,8 +11386,6 @@ snapshots:'@walletconnect/window-getters': 1.0.1tslib: 1.14.1- '@xmldom/[email protected]': {}-@@ -11667,8 +11616,6 @@ snapshots:dependencies:file-uri-to-path: 1.0.0- [email protected]: {}-@@ -12107,8 +12054,6 @@ snapshots:- [email protected]: {}-dependencies:domelementtype: 2.3.0@@ -12129,10 +12074,6 @@ snapshots:- dependencies:- underscore: 1.13.6-dependencies:call-bind-apply-helpers: 1.0.2@@ -12811,8 +12752,6 @@ snapshots:- [email protected]: {}-dependencies:acorn: 8.15.0@@ -13420,13 +13359,6 @@ snapshots:chalk: 5.6.0diff-match-patch: 1.0.5- dependencies:- lie: 3.3.0- pako: 1.0.11- readable-stream: 2.3.8- setimmediate: 1.0.5-dependencies:buffer-equal-constant-time: 1.0.1@@ -13494,10 +13426,6 @@ snapshots:nan: 2.22.2optional: true- dependencies:- immediate: 3.0.6-@@ -13604,12 +13532,6 @@ snapshots:dependencies:js-tokens: 4.0.0- dependencies:- duck: 0.1.12- option: 0.2.4- underscore: 1.13.6-@@ -13630,19 +13552,6 @@ snapshots:dependencies:tmpl: 1.0.5- dependencies:- '@xmldom/xmldom': 0.8.10- argparse: 1.0.10- base64-js: 1.5.1- bluebird: 3.4.7- dingbat-to-unicode: 1.0.1- jszip: 3.10.1- lop: 0.4.1- path-is-absolute: 1.0.1- underscore: 1.13.6- xmlbuilder: 10.1.1-@@ -13872,8 +13781,6 @@ snapshots:ws: 8.18.0([email protected])([email protected])zod: 3.24.2- [email protected]: {}-dependencies:'@adraffy/ens-normalize': 1.11.0@@ -14006,8 +13913,6 @@ snapshots:- [email protected]: {}-@@ -14510,8 +14415,6 @@ snapshots:gopd: 1.0.1has-property-descriptors: 1.0.2- [email protected]: {}-@@ -14934,8 +14837,6 @@ snapshots:- [email protected]: {}-@@ -15358,8 +15259,6 @@ snapshots:sax: 1.4.1xmlbuilder: 11.0.1- [email protected]: {}-diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 77a294295..100c55f7c 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -1061,6 +1061,22 @@ export async function performSummary(return document;}+ if (document.markdown === undefined) {+ document.warning =+ "Summary mode is not supported without the markdown format." ++ (document.warning ? " " + document.warning : "");+ return document;+ }++ const trimOutput = trimToTokenLimit(+ document.markdown!,+ 120000,+ "gpt-4o-mini",+ document.warning,+ );++ document.warning = trimOutput.warning;+const generationOptions: GenerateCompletionsOptions = {logger: meta.logger.child({method: "performSummary/generateCompletions",@@ -1079,10 +1095,10 @@ export async function performSummary(required: ["summary"],},},- markdown: document.markdown,+ markdown: trimOutput.text,previousWarning: document.warning,model: getModel("gpt-4o-mini", "openai"),- retryModel: getModel("gpt-5", "openai"),+ retryModel: getModel("gpt-4o", "openai"),costTrackingOptions: {costTracking: meta.costTracking,metadata: {