Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix encoding if error
rafaelsideguideaed11e72a60e
diff --git a/apps/api/src/lib/LLM-extraction/helpers.ts b/apps/api/src/lib/LLM-extraction/helpers.tsindex f47a6b3c1..2143a32db 100644--- a/apps/api/src/lib/LLM-extraction/helpers.ts+++ b/apps/api/src/lib/LLM-extraction/helpers.ts@@ -6,7 +6,13 @@ export function numTokensFromString(message: string, model: string): number {const encoder = encoding_for_model(model as TiktokenModel);// Encode the message into tokens- const tokens = encoder.encode(message);+ let tokens: Uint32Array;+ try {+ tokens = encoder.encode(message);+ } catch (error) {+ message = message.replace("<|endoftext|>", "");+ tokens = encoder.encode(message);+ }// Free the encoder resources after useencoder.free();