Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update extract-redis.ts

Nicolasc323c6467173
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts
index cb793572a..0df700f46 100644
--- a/apps/api/src/lib/extract/extract-redis.ts
+++ b/apps/api/src/lib/extract/extract-redis.ts
@@ -54,6 +54,19 @@ export async function updateExtract(
extract.steps = [...current.steps, ...extract.steps];
}
+ // Limit links in steps to 500
+ if (extract.steps) {
+ extract.steps = extract.steps.map(step => {
+ if (step.discoveredLinks && step.discoveredLinks.length > 500) {
+ return {
+ ...step,
+ discoveredLinks: step.discoveredLinks.slice(0, 500)
+ };
+ }
+ return step;
+ });
+ }
+
await redisConnection.set(
"extract:" + id,
JSON.stringify({ ...current, ...extract }),