Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): optimize redis calls (#2564)
tome77e43bce8d0
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 3f102d7e0..84af3fe96 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -38,9 +38,12 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {zeroDataRetention: crawl.zeroDataRetention,});- await redisEvictConnection.set("crawl:" + id, JSON.stringify(crawl));- await redisEvictConnection.expire("crawl:" + id, 24 * 60 * 60);-+ await redisEvictConnection.set(+ "crawl:" + id,+ JSON.stringify(crawl),+ "EX",+ 24 * 60 * 60,+ );await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);await redisEvictConnection.expire("crawls_by_team_id:" + crawl.team_id,@@ -114,13 +117,12 @@ export async function addCrawlJob(crawlId: id,});- await redisEvictConnection.sadd("crawl:" + id + ":jobs", job_id);- await redisEvictConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60);- await redisEvictConnection.sadd("crawl:" + id + ":jobs_qualified", job_id);- await redisEvictConnection.expire(- "crawl:" + id + ":jobs_qualified",- 24 * 60 * 60,- );+ const pipeline = redisEvictConnection.pipeline();+ pipeline.sadd("crawl:" + id + ":jobs", job_id);+ pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);+ pipeline.sadd("crawl:" + id + ":jobs_qualified", job_id);+ pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);+ await pipeline.exec();});}@@ -137,16 +139,12 @@ export async function addCrawlJobs(method: "addCrawlJobs",crawlId: id,});- await redisEvictConnection.sadd("crawl:" + id + ":jobs", ...job_ids);- await redisEvictConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60);- await redisEvictConnection.sadd(- "crawl:" + id + ":jobs_qualified",- ...job_ids,- );- await redisEvictConnection.expire(- "crawl:" + id + ":jobs_qualified",- 24 * 60 * 60,- );+ const pipeline = redisEvictConnection.pipeline();+ pipeline.sadd("crawl:" + id + ":jobs", ...job_ids);+ pipeline.expire("crawl:" + id + ":jobs", 24 * 60 * 60);+ pipeline.sadd("crawl:" + id + ":jobs_qualified", ...job_ids);+ pipeline.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);+ await pipeline.exec();}export async function addCrawlJobDone(@@ -161,27 +159,19 @@ export async function addCrawlJobDone(method: "addCrawlJobDone",crawlId: id,});- await redisEvictConnection.sadd("crawl:" + id + ":jobs_done", job_id);- await redisEvictConnection.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);+ const pipeline = redisEvictConnection.pipeline();+ pipeline.sadd("crawl:" + id + ":jobs_done", job_id);+ pipeline.expire("crawl:" + id + ":jobs_done", 24 * 60 * 60);if (success) {- await redisEvictConnection.zadd(- "crawl:" + id + ":jobs_donez_ordered",- Date.now(),- job_id,- );+ pipeline.zadd("crawl:" + id + ":jobs_donez_ordered", Date.now(), job_id);} else {// in case it's already been pushed, make sure it's removed- await redisEvictConnection.zrem(- "crawl:" + id + ":jobs_donez_ordered",- job_id,- );+ pipeline.zrem("crawl:" + id + ":jobs_donez_ordered", job_id);}- await redisEvictConnection.expire(- "crawl:" + id + ":jobs_donez_ordered",- 24 * 60 * 60,- );+ pipeline.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);+ await pipeline.exec();}export async function getDoneJobsOrderedLength(@@ -275,8 +265,13 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {method: "finishCrawl",crawlId: id,});- await redisEvictConnection.set("crawl:" + id + ":finish", "yes");- await redisEvictConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);++ await redisEvictConnection.set(+ "crawl:" + id + ":finish",+ "yes",+ "EX",+ 24 * 60 * 60,+ );await redisEvictConnection.srem("active_crawls", id);@@ -431,33 +426,26 @@ export async function lockURL(}}- let res: boolean;+ const pipeline = redisEvictConnection.pipeline();+if (!sc.crawlerOptions?.deduplicateSimilarURLs) {- res =- (await redisEvictConnection.sadd(- "crawl:" + id + ":visited",- normalizedUrl,- )) !== 0;+ pipeline.sadd("crawl:" + id + ":visited", normalizedUrl);} else {const permutation = generateURLPermutations(normalizedUrl)[0].href;- const x = await redisEvictConnection.sadd(- "crawl:" + id + ":visited",- permutation,- );- res = x !== 0;+ pipeline.sadd("crawl:" + id + ":visited", permutation);}- await redisEvictConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60);+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);++ const results = await pipeline.exec();+ const saddResult = results?.[0]?.[1] as number;+ const res = saddResult !== 0;if (res) {- await redisEvictConnection.sadd(- "crawl:" + id + ":visited_unique",- normalizedUrl,- );- await redisEvictConnection.expire(- "crawl:" + id + ":visited_unique",- 24 * 60 * 60,- );+ const uniquePipeline = redisEvictConnection.pipeline();+ uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);+ uniquePipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);+ await uniquePipeline.exec();}setSpanAttributes(span, { "crawl.url_locked": res });@@ -484,32 +472,26 @@ export async function lockURLs(// Add to visited_unique setlogger.debug("Locking " + urls.length + " URLs...");- await redisEvictConnection.sadd("crawl:" + id + ":visited_unique", ...urls);- await redisEvictConnection.expire(- "crawl:" + id + ":visited_unique",- 24 * 60 * 60,- );- let res: boolean;+ const pipeline = redisEvictConnection.pipeline();+ pipeline.sadd("crawl:" + id + ":visited_unique", ...urls);+ pipeline.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60);+if (!sc.crawlerOptions?.deduplicateSimilarURLs) {- const x = await redisEvictConnection.sadd(- "crawl:" + id + ":visited",- ...urls,- );- res = x === urls.length;+ pipeline.sadd("crawl:" + id + ":visited", ...urls);} else {const allPermutations = urls.map(url => generateURLPermutations(url)[0].href,);logger.debug("Adding " + allPermutations.length + " URL permutations...");- const x = await redisEvictConnection.sadd(- "crawl:" + id + ":visited",- ...allPermutations,- );- res = x === allPermutations.length;+ pipeline.sadd("crawl:" + id + ":visited", ...allPermutations);}- await redisEvictConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60);+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);++ const results = await pipeline.exec();+ const saddResult = results?.[2]?.[1] as number;+ const res = saddResult === urls.length;logger.debug("lockURLs final result: " + res, { res });return res;diff --git a/apps/api/src/lib/deep-research/deep-research-redis.ts b/apps/api/src/lib/deep-research/deep-research-redis.tsindex 0135a0f04..d9d4e8d82 100644--- a/apps/api/src/lib/deep-research/deep-research-redis.ts+++ b/apps/api/src/lib/deep-research/deep-research-redis.ts@@ -55,8 +55,9 @@ export async function saveDeepResearch(await redisEvictConnection.set("deep-research:" + id,JSON.stringify(research),+ "EX",+ DEEP_RESEARCH_TTL,);- await redisEvictConnection.expire("deep-research:" + id, DEEP_RESEARCH_TTL);}export async function getDeepResearch(@@ -97,8 +98,9 @@ export async function updateDeepResearch(await redisEvictConnection.set("deep-research:" + id,JSON.stringify(updatedResearch),+ "EX",+ DEEP_RESEARCH_TTL,);- await redisEvictConnection.expire("deep-research:" + id, DEEP_RESEARCH_TTL);}export async function getDeepResearchExpiry(id: string): Promise<Date> {diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.tsindex 3b153e403..0ab06ab0d 100644--- a/apps/api/src/lib/extract/extract-redis.ts+++ b/apps/api/src/lib/extract/extract-redis.ts@@ -69,8 +69,9 @@ export async function saveExtract(id: string, extract: StoredExtract) {await redisEvictConnection.set("extract:" + id,JSON.stringify(minimalExtract),+ "EX",+ EXTRACT_TTL,);- await redisEvictConnection.expire("extract:" + id, EXTRACT_TTL);}export async function getExtract(id: string): Promise<StoredExtract | null> {@@ -131,8 +132,9 @@ export async function updateExtract(await redisEvictConnection.set("extract:" + id,JSON.stringify(minimalExtract),+ "EX",+ EXTRACT_TTL,);- await redisEvictConnection.expire("extract:" + id, EXTRACT_TTL);}export async function getExtractExpiry(id: string): Promise<Date> {diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.tsindex 8eab4e4aa..b92937631 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts@@ -23,8 +23,12 @@ export async function saveGeneratedLlmsTxt(data: GenerationData,): Promise<void> {_logger.debug("Saving llmstxt generation " + id + " to Redis...");- await redisEvictConnection.set("generation:" + id, JSON.stringify(data));- await redisEvictConnection.expire("generation:" + id, GENERATION_TTL);+ await redisEvictConnection.set(+ "generation:" + id,+ JSON.stringify(data),+ "EX",+ GENERATION_TTL,+ );}export async function getGeneratedLlmsTxt(@@ -49,8 +53,9 @@ export async function updateGeneratedLlmsTxt(await redisEvictConnection.set("generation:" + id,JSON.stringify(updatedGeneration),+ "EX",+ GENERATION_TTL,);- await redisEvictConnection.expire("generation:" + id, GENERATION_TTL);}export async function getGeneratedLlmsTxtExpiry(id: string): Promise<Date> {diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex deea0e867..30b2aff72 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -4,7 +4,9 @@ import { RateLimiterMode } from "../types";import Redis from "ioredis";import type { AuthCreditUsageChunk } from "../controllers/v1/types";-export const redisRateLimitClient = new Redis(config.REDIS_RATE_LIMIT_URL!);+export const redisRateLimitClient = new Redis(config.REDIS_RATE_LIMIT_URL!, {+ enableAutoPipelining: true,+});const createRateLimiter = (keyPrefix, points) =>new RateLimiterRedis({diff --git a/apps/api/src/services/redis.ts b/apps/api/src/services/redis.tsindex 7049020c0..f31493fad 100644--- a/apps/api/src/services/redis.ts+++ b/apps/api/src/services/redis.ts@@ -73,4 +73,6 @@ const deleteKey = async (key: string) => {export { setValue, getValue, deleteKey };const redisEvictURL = config.REDIS_EVICT_URL ?? config.REDIS_RATE_LIMIT_URL;-export const redisEvictConnection = new IORedis(redisEvictURL!);+export const redisEvictConnection = new IORedis(redisEvictURL!, {+ enableAutoPipelining: true,+});diff --git a/apps/api/src/services/worker/redis.ts b/apps/api/src/services/worker/redis.tsindex c98859152..0c4f58d67 100644--- a/apps/api/src/services/worker/redis.ts+++ b/apps/api/src/services/worker/redis.ts@@ -62,6 +62,7 @@ const redis = new IORedis(config.REDIS_URL!, {lazyConnect: true,maxRetriesPerRequest: null,enableReadyCheck: false,+ enableAutoPipelining: true,});let initPromise: Promise<void> | null = null;