Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update extract-redis.ts
Nicolasc323c6467173
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.tsindex cb793572a..0df700f46 100644--- a/apps/api/src/lib/extract/extract-redis.ts+++ b/apps/api/src/lib/extract/extract-redis.ts@@ -54,6 +54,19 @@ export async function updateExtract(extract.steps = [...current.steps, ...extract.steps];}+ // Limit links in steps to 500+ if (extract.steps) {+ extract.steps = extract.steps.map(step => {+ if (step.discoveredLinks && step.discoveredLinks.length > 500) {+ return {+ ...step,+ discoveredLinks: step.discoveredLinks.slice(0, 500)+ };+ }+ return step;+ });+ }+await redisConnection.set("extract:" + id,JSON.stringify({ ...current, ...extract }),