Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix encoding if error

rafaelsideguideaed11e72a60e
diff --git a/apps/api/src/lib/LLM-extraction/helpers.ts b/apps/api/src/lib/LLM-extraction/helpers.ts
index f47a6b3c1..2143a32db 100644
--- a/apps/api/src/lib/LLM-extraction/helpers.ts
+++ b/apps/api/src/lib/LLM-extraction/helpers.ts
@@ -6,7 +6,13 @@ export function numTokensFromString(message: string, model: string): number {
const encoder = encoding_for_model(model as TiktokenModel);
// Encode the message into tokens
- const tokens = encoder.encode(message);
+ let tokens: Uint32Array;
+ try {
+ tokens = encoder.encode(message);
+ } catch (error) {
+ message = message.replace("<|endoftext|>", "");
+ tokens = encoder.encode(message);
+ }
// Free the encoder resources after use
encoder.free();