Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): optimize redis calls (#2564)

tome77e43bce8d0
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 3f102d7e0..84af3fe96 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -38,9 +38,12 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
zeroDataRetention: crawl.zeroDataRetention,
});
- await redisEvictConnection.set("crawl:" + id, JSON.stringify(crawl));
- await redisEvictConnection.expire("crawl:" + id, 24 * 60 * 60);
-
+ await redisEvictConnection.set(
+ "crawl:" + id,
+ JSON.stringify(crawl),
+ "EX",
+ 24 * 60 * 60,
+ );
await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);
await redisEvictConnection.expire(
"crawls_by_team_id:" + crawl.team_id,
@@ -114,13 +117,12 @@ export async function addCrawlJob(
crawlId: id,
});
- await redisEvictConnection.sadd("crawl:" + id + ":jobs", job_id);
- await redisEvictConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
- await redisEvictConnection.sadd("crawl:" + id + ":jobs_qualified", job_id);
- await redisEvictConnection.expire(
- "crawl:" + id + ":jobs_qualified",
- 24 * 60 * 60,
- );
+ const pipeline = redisEvictConnection.pipeline();
+ pipeline.sadd("crawl:" + id + ":jobs", job_id);
+ pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
+ pipeline.sadd("crawl:" + id + ":jobs_qualified", job_id);
+ pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);
+ await pipeline.exec();
});
}
@@ -137,16 +139,12 @@ export async function addCrawlJobs(
method: "addCrawlJobs",
crawlId: id,
});
- await redisEvictConnection.sadd("crawl:" + id + ":jobs", ...job_ids);
- await redisEvictConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
- await redisEvictConnection.sadd(
- "crawl:" + id + ":jobs_qualified",
- ...job_ids,
- );
- await redisEvictConnection.expire(
- "crawl:" + id + ":jobs_qualified",
- 24 * 60 * 60,
- );
+ const pipeline = redisEvictConnection.pipeline();
+ pipeline.sadd("crawl:" + id + ":jobs", ...job_ids);
+ pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
+ pipeline.sadd("crawl:" + id + ":jobs_qualified", ...job_ids);
+ pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);
+ await pipeline.exec();
}
export async function addCrawlJobDone(
@@ -161,27 +159,19 @@ export async function addCrawlJobDone(
method: "addCrawlJobDone",
crawlId: id,
});
- await redisEvictConnection.sadd("crawl:" + id + ":jobs_done", job_id);
- await redisEvictConnection.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);
+ const pipeline = redisEvictConnection.pipeline();
+ pipeline.sadd("crawl:" + id + ":jobs_done", job_id);
+ pipeline.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);
if (success) {
- await redisEvictConnection.zadd(
- "crawl:" + id + ":jobs_donez_ordered",
- Date.now(),
- job_id,
- );
+ pipeline.zadd("crawl:" + id + ":jobs_donez_ordered", Date.now(), job_id);
} else {
// in case it's already been pushed, make sure it's removed
- await redisEvictConnection.zrem(
- "crawl:" + id + ":jobs_donez_ordered",
- job_id,
- );
+ pipeline.zrem("crawl:" + id + ":jobs_donez_ordered", job_id);
}
- await redisEvictConnection.expire(
- "crawl:" + id + ":jobs_donez_ordered",
- 24 * 60 * 60,
- );
+ pipeline.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);
+ await pipeline.exec();
}
export async function getDoneJobsOrderedLength(
@@ -275,8 +265,13 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {
method: "finishCrawl",
crawlId: id,
});
- await redisEvictConnection.set("crawl:" + id + ":finish", "yes");
- await redisEvictConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);
+
+ await redisEvictConnection.set(
+ "crawl:" + id + ":finish",
+ "yes",
+ "EX",
+ 24 * 60 * 60,
+ );
await redisEvictConnection.srem("active_crawls", id);
@@ -431,33 +426,26 @@ export async function lockURL(
}
}
- let res: boolean;
+ const pipeline = redisEvictConnection.pipeline();
+
if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
- res =
- (await redisEvictConnection.sadd(
- "crawl:" + id + ":visited",
- normalizedUrl,
- )) !== 0;
+ pipeline.sadd("crawl:" + id + ":visited", normalizedUrl);
} else {
const permutation = generateURLPermutations(normalizedUrl)[0].href;
- const x = await redisEvictConnection.sadd(
- "crawl:" + id + ":visited",
- permutation,
- );
- res = x !== 0;
+ pipeline.sadd("crawl:" + id + ":visited", permutation);
}
- await redisEvictConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+
+ const results = await pipeline.exec();
+ const saddResult = results?.[0]?.[1] as number;
+ const res = saddResult !== 0;
if (res) {
- await redisEvictConnection.sadd(
- "crawl:" + id + ":visited_unique",
- normalizedUrl,
- );
- await redisEvictConnection.expire(
- "crawl:" + id + ":visited_unique",
- 24 * 60 * 60,
- );
+ const uniquePipeline = redisEvictConnection.pipeline();
+ uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);
+ uniquePipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);
+ await uniquePipeline.exec();
}
setSpanAttributes(span, { "crawl.url_locked": res });
@@ -484,32 +472,26 @@ export async function lockURLs(
// Add to visited_unique set
logger.debug("Locking " + urls.length + " URLs...");
- await redisEvictConnection.sadd("crawl:" + id + ":visited_unique", ...urls);
- await redisEvictConnection.expire(
- "crawl:" + id + ":visited_unique",
- 24 * 60 * 60,
- );
- let res: boolean;
+ const pipeline = redisEvictConnection.pipeline();
+ pipeline.sadd("crawl:" + id + ":visited_unique", ...urls);
+ pipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);
+
if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
- const x = await redisEvictConnection.sadd(
- "crawl:" + id + ":visited",
- ...urls,
- );
- res = x === urls.length;
+ pipeline.sadd("crawl:" + id + ":visited", ...urls);
} else {
const allPermutations = urls.map(
url => generateURLPermutations(url)[0].href,
);
logger.debug("Adding " + allPermutations.length + " URL permutations...");
- const x = await redisEvictConnection.sadd(
- "crawl:" + id + ":visited",
- ...allPermutations,
- );
- res = x === allPermutations.length;
+ pipeline.sadd("crawl:" + id + ":visited", ...allPermutations);
}
- await redisEvictConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);
+
+ const results = await pipeline.exec();
+ const saddResult = results?.[2]?.[1] as number;
+ const res = saddResult === urls.length;
logger.debug("lockURLs final result: " + res, { res });
return res;
diff --git a/apps/api/src/lib/deep-research/deep-research-redis.ts b/apps/api/src/lib/deep-research/deep-research-redis.ts
index 0135a0f04..d9d4e8d82 100644
--- a/apps/api/src/lib/deep-research/deep-research-redis.ts
+++ b/apps/api/src/lib/deep-research/deep-research-redis.ts
@@ -55,8 +55,9 @@ export async function saveDeepResearch(
await redisEvictConnection.set(
"deep-research:" + id,
JSON.stringify(research),
+ "EX",
+ DEEP_RESEARCH_TTL,
);
- await redisEvictConnection.expire("deep-research:" + id, DEEP_RESEARCH_TTL);
}
export async function getDeepResearch(
@@ -97,8 +98,9 @@ export async function updateDeepResearch(
await redisEvictConnection.set(
"deep-research:" + id,
JSON.stringify(updatedResearch),
+ "EX",
+ DEEP_RESEARCH_TTL,
);
- await redisEvictConnection.expire("deep-research:" + id, DEEP_RESEARCH_TTL);
}
export async function getDeepResearchExpiry(id: string): Promise<Date> {
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts
index 3b153e403..0ab06ab0d 100644
--- a/apps/api/src/lib/extract/extract-redis.ts
+++ b/apps/api/src/lib/extract/extract-redis.ts
@@ -69,8 +69,9 @@ export async function saveExtract(id: string, extract: StoredExtract) {
await redisEvictConnection.set(
"extract:" + id,
JSON.stringify(minimalExtract),
+ "EX",
+ EXTRACT_TTL,
);
- await redisEvictConnection.expire("extract:" + id, EXTRACT_TTL);
}
export async function getExtract(id: string): Promise<StoredExtract | null> {
@@ -131,8 +132,9 @@ export async function updateExtract(
await redisEvictConnection.set(
"extract:" + id,
JSON.stringify(minimalExtract),
+ "EX",
+ EXTRACT_TTL,
);
- await redisEvictConnection.expire("extract:" + id, EXTRACT_TTL);
}
export async function getExtractExpiry(id: string): Promise<Date> {
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts
index 8eab4e4aa..b92937631 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts
@@ -23,8 +23,12 @@ export async function saveGeneratedLlmsTxt(
data: GenerationData,
): Promise<void> {
_logger.debug("Saving llmstxt generation " + id + " to Redis...");
- await redisEvictConnection.set("generation:" + id, JSON.stringify(data));
- await redisEvictConnection.expire("generation:" + id, GENERATION_TTL);
+ await redisEvictConnection.set(
+ "generation:" + id,
+ JSON.stringify(data),
+ "EX",
+ GENERATION_TTL,
+ );
}
export async function getGeneratedLlmsTxt(
@@ -49,8 +53,9 @@ export async function updateGeneratedLlmsTxt(
await redisEvictConnection.set(
"generation:" + id,
JSON.stringify(updatedGeneration),
+ "EX",
+ GENERATION_TTL,
);
- await redisEvictConnection.expire("generation:" + id, GENERATION_TTL);
}
export async function getGeneratedLlmsTxtExpiry(id: string): Promise<Date> {
diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts
index deea0e867..30b2aff72 100644
--- a/apps/api/src/services/rate-limiter.ts
+++ b/apps/api/src/services/rate-limiter.ts
@@ -4,7 +4,9 @@ import { RateLimiterMode } from "../types";
import Redis from "ioredis";
import type { AuthCreditUsageChunk } from "../controllers/v1/types";
-export const redisRateLimitClient = new Redis(config.REDIS_RATE_LIMIT_URL!);
+export const redisRateLimitClient = new Redis(config.REDIS_RATE_LIMIT_URL!, {
+ enableAutoPipelining: true,
+});
const createRateLimiter = (keyPrefix, points) =>
new RateLimiterRedis({
diff --git a/apps/api/src/services/redis.ts b/apps/api/src/services/redis.ts
index 7049020c0..f31493fad 100644
--- a/apps/api/src/services/redis.ts
+++ b/apps/api/src/services/redis.ts
@@ -73,4 +73,6 @@ const deleteKey = async (key: string) => {
export { setValue, getValue, deleteKey };
const redisEvictURL = config.REDIS_EVICT_URL ?? config.REDIS_RATE_LIMIT_URL;
-export const redisEvictConnection = new IORedis(redisEvictURL!);
+export const redisEvictConnection = new IORedis(redisEvictURL!, {
+ enableAutoPipelining: true,
+});
diff --git a/apps/api/src/services/worker/redis.ts b/apps/api/src/services/worker/redis.ts
index c98859152..0c4f58d67 100644
--- a/apps/api/src/services/worker/redis.ts
+++ b/apps/api/src/services/worker/redis.ts
@@ -62,6 +62,7 @@ const redis = new IORedis(config.REDIS_URL!, {
lazyConnect: true,
maxRetriesPerRequest: null,
enableReadyCheck: false,
+ enableAutoPipelining: true,
});
let initPromise: Promise<void> | null = null;