Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(extractSmartScrape): better pagination handling

Gergő Móricz751c30f139be
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 0e7ac878f..37205343e 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -16,7 +16,7 @@ const commonSmartScrapeProperties = {
shouldUseSmartscrape: {
type: "boolean",
description:
- "Set to `true` if any of the extractedData is null and you think you can find the information by performing user-like interactions (e.g., clicking buttons/accordions to reveal hidden text, login, inputs etc.). SmartScrape can perform these actions to access the data.",
+ "Set to `true` if any of the extractedData is null and you think you can find the information by performing user-like interactions (e.g., clicking buttons/accordions to reveal hidden text, login, inputs, pagination etc.). SmartScrape can perform these actions to access the data.",
},
// Note: extractedData is added dynamically in prepareSmartScrapeSchema
};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 21572c9d1..73456096c 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -386,7 +386,7 @@ export async function generateCompletions({
const repairConfig = {
experimental_repairText: async ({ text, error }) => {
// AI may output a markdown JSON code block. Remove it - mogery
- logger.debug("Repairing text", { textType: typeof text, error });
+ logger.debug("Repairing text", { textType: typeof text, textPeek: JSON.stringify(text).slice(0, 100) + "...", error });
if (typeof text === "string" && text.trim().startsWith("```")) {
if (text.trim().startsWith("```json")) {