Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(summary): revert to gpt-4o-mini but trim tokens correctly

Gergő Móriczfb590d38e057
diff --git a/apps/api/package.json b/apps/api/package.json
index 3cf01b00a..95c0f22e2 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -112,7 +112,6 @@
"langfuse": "^3.38.4",
"langfuse-vercel": "^3.38.4",
"lodash": "^4.17.21",
- "mammoth": "^1.7.2",
"marked": "^14.1.2",
"ollama-ai-provider": "^1.2.0",
"openai": "^5.20.2",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 6b4dc99ac..c10b5de23 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -165,9 +165,6 @@ importers:
lodash:
specifier: ^4.17.21
version: 4.17.21
- mammoth:
- specifier: ^1.7.2
- version: 1.7.2
marked:
specifier: ^14.1.2
version: 14.1.2
@@ -3431,10 +3428,6 @@ packages:
'@walletconnect/[email protected]':
resolution: {integrity: sha512-9koTqyGrM2cqFRW517BPY/iEtUDx2r1+Pwwu5m7sJ7ka79wi3EyqhqcICk/yDmv6jAS1rjKgTKXlEhanYjijcA==}
- '@xmldom/[email protected]':
- resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==}
- engines: {node: '>=10.0.0'}
-
resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -3656,9 +3649,6 @@ packages:
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
- resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==}
-
resolution: {integrity: sha512-v2YAxEmKaBLahNwE1mjp4WON6huMNeuDvagFZW+ASCuA/ku0bXR9hSMw0XpiqMoA3+rmnyck/tPRSFQkoC9Cuw==}
@@ -4113,9 +4103,6 @@ packages:
resolution: {integrity: sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA==}
- resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==}
-
resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}
@@ -4133,9 +4120,6 @@ packages:
resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}
engines: {node: '>=12'}
- resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==}
-
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
@@ -4690,9 +4674,6 @@ packages:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
- resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}
-
resolution: {integrity: sha512-5tCuY9BV8ujfOpwtAGgsTx9CGUapcFMEEyByLv1B+v2+6DhAcw+Zr0nhQT7uwaZ7DiourxFEscghOR8e1aPLQw==}
@@ -5087,9 +5068,6 @@ packages:
engines: {node: ^18.0.0 || >=20.0.0}
hasBin: true
- resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==}
-
resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==}
@@ -5145,9 +5123,6 @@ packages:
resolution: {integrity: sha512-4lSWmly2Nsj3LaTxxtFmJWuP3Kx+0hYHEd+aNrcXEWT0nKWaPd9/QZPiMkkC680zeALFGHQdQWjBvnilL+vgWA==}
- resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==}
-
resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==}
engines: {node: '>=14'}
@@ -5234,9 +5209,6 @@ packages:
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
- resolution: {integrity: sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==}
-
resolution: {integrity: sha512-CQl19J/g+Hbjbv4Y3mFNNXFEL/5t/KCg8POCuUqd4rMKjGG+j1ybER83hxV58zL+dFI1PTkt3GNFSHRt+d8qEQ==}
engines: {node: 14 || >=16.14}
@@ -5261,11 +5233,6 @@ packages:
resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==}
- resolution: {integrity: sha512-MqWU2hcLf1I5QMKyAbfJCvrLxnv5WztrAQyorfZ+WPq7Hk82vZFmvfR2/64ajIPpM4jlq0TXp1xZvp/FFaL1Ug==}
- engines: {node: '>=12.0.0'}
- hasBin: true
-
resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}
@@ -5557,9 +5524,6 @@ packages:
zod:
optional: true
- resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}
-
resolution: {integrity: sha512-17Gk/eFsFRAZ80p5eKqv89a57uXjd3NgIf1CaXojATPBuujVc/fQSVhBeAU9JCRB+k7J50WQAyWTxK19T9GgbA==}
peerDependencies:
@@ -5624,9 +5588,6 @@ packages:
resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==}
- resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
-
resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}
@@ -6113,9 +6074,6 @@ packages:
resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==}
engines: {node: '>= 0.4'}
- resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==}
-
resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
@@ -6542,9 +6500,6 @@ packages:
resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==}
- resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}
-
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
@@ -6890,10 +6845,6 @@ packages:
resolution: {integrity: sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==}
engines: {node: '>=4.0.0'}
- resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==}
- engines: {node: '>=4.0'}
-
resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==}
engines: {node: '>=4.0'}
@@ -11435,8 +11386,6 @@ snapshots:
'@walletconnect/window-getters': 1.0.1
tslib: 1.14.1
- '@xmldom/[email protected]': {}
-
@@ -11667,8 +11616,6 @@ snapshots:
dependencies:
file-uri-to-path: 1.0.0
-
@@ -12107,8 +12054,6 @@ snapshots:
-
dependencies:
domelementtype: 2.3.0
@@ -12129,10 +12074,6 @@ snapshots:
- dependencies:
- underscore: 1.13.6
-
dependencies:
call-bind-apply-helpers: 1.0.2
@@ -12811,8 +12752,6 @@ snapshots:
-
dependencies:
acorn: 8.15.0
@@ -13420,13 +13359,6 @@ snapshots:
chalk: 5.6.0
diff-match-patch: 1.0.5
- dependencies:
- lie: 3.3.0
- pako: 1.0.11
- readable-stream: 2.3.8
- setimmediate: 1.0.5
-
dependencies:
buffer-equal-constant-time: 1.0.1
@@ -13494,10 +13426,6 @@ snapshots:
nan: 2.22.2
optional: true
- dependencies:
- immediate: 3.0.6
-
@@ -13604,12 +13532,6 @@ snapshots:
dependencies:
js-tokens: 4.0.0
- dependencies:
- duck: 0.1.12
- option: 0.2.4
- underscore: 1.13.6
-
@@ -13630,19 +13552,6 @@ snapshots:
dependencies:
tmpl: 1.0.5
- dependencies:
- '@xmldom/xmldom': 0.8.10
- argparse: 1.0.10
- base64-js: 1.5.1
- bluebird: 3.4.7
- dingbat-to-unicode: 1.0.1
- jszip: 3.10.1
- lop: 0.4.1
- path-is-absolute: 1.0.1
- underscore: 1.13.6
- xmlbuilder: 10.1.1
-
@@ -13872,8 +13781,6 @@ snapshots:
zod: 3.24.2
-
dependencies:
'@adraffy/ens-normalize': 1.11.0
@@ -14006,8 +13913,6 @@ snapshots:
-
@@ -14510,8 +14415,6 @@ snapshots:
gopd: 1.0.1
has-property-descriptors: 1.0.2
-
@@ -14934,8 +14837,6 @@ snapshots:
-
@@ -15358,8 +15259,6 @@ snapshots:
sax: 1.4.1
xmlbuilder: 11.0.1
-
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 77a294295..100c55f7c 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -1061,6 +1061,22 @@ export async function performSummary(
return document;
}
+ if (document.markdown === undefined) {
+ document.warning =
+ "Summary mode is not supported without the markdown format." +
+ (document.warning ? " " + document.warning : "");
+ return document;
+ }
+
+ const trimOutput = trimToTokenLimit(
+ document.markdown!,
+ 120000,
+ "gpt-4o-mini",
+ document.warning,
+ );
+
+ document.warning = trimOutput.warning;
+
const generationOptions: GenerateCompletionsOptions = {
logger: meta.logger.child({
method: "performSummary/generateCompletions",
@@ -1079,10 +1095,10 @@ export async function performSummary(
required: ["summary"],
},
},
- markdown: document.markdown,
+ markdown: trimOutput.text,
previousWarning: document.warning,
model: getModel("gpt-4o-mini", "openai"),
- retryModel: getModel("gpt-5", "openai"),
+ retryModel: getModel("gpt-4o", "openai"),
costTrackingOptions: {
costTracking: meta.costTracking,
metadata: {