Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(llmExtract): more logging

Gergő Móricz509e6e658c9b
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 041fd036b..21572c9d1 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -386,6 +386,8 @@ export async function generateCompletions({
const repairConfig = {
experimental_repairText: async ({ text, error }) => {
// AI may output a markdown JSON code block. Remove it - mogery
+ logger.debug("Repairing text", { textType: typeof text, error });
+
if (typeof text === "string" && text.trim().startsWith("```")) {
if (text.trim().startsWith("```json")) {
text = text.trim().slice("```json".length).trim();
@@ -400,6 +402,7 @@ export async function generateCompletions({
// If this fixes the JSON, just return it. If not, continue - mogery
try {
JSON.parse(text);
+ logger.debug("Repaired text with string manipulation");
return text;
} catch (e) {
logger.error("Even after repairing, failed to parse JSON", { error: e });
@@ -418,6 +421,7 @@ export async function generateCompletions({
},
},
});
+ logger.debug("Repaired text with LLM");
return fixedText;
} catch (repairError) {
lastError = repairError as Error;