Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(llmExtract): more logging
Gergő Móricz509e6e658c9b
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 041fd036b..21572c9d1 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -386,6 +386,8 @@ export async function generateCompletions({const repairConfig = {experimental_repairText: async ({ text, error }) => {// AI may output a markdown JSON code block. Remove it - mogery+ logger.debug("Repairing text", { textType: typeof text, error });+if (typeof text === "string" && text.trim().startsWith("```")) {if (text.trim().startsWith("```json")) {text = text.trim().slice("```json".length).trim();@@ -400,6 +402,7 @@ export async function generateCompletions({// If this fixes the JSON, just return it. If not, continue - mogerytry {JSON.parse(text);+ logger.debug("Repaired text with string manipulation");return text;} catch (e) {logger.error("Even after repairing, failed to parse JSON", { error: e });@@ -418,6 +421,7 @@ export async function generateCompletions({},},});+ logger.debug("Repaired text with LLM");return fixedText;} catch (repairError) {lastError = repairError as Error;