Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Refactor redisUsEast to redisGlobal (#3556)

Steven Tey250809893caf
diff --git a/apps/web/app/(ee)/api/track/click/route.ts b/apps/web/app/(ee)/api/track/click/route.ts
index 18bef39aa2..466dcde5ea 100644
--- a/apps/web/app/(ee)/api/track/click/route.ts
+++ b/apps/web/app/(ee)/api/track/click/route.ts
@@ -14,7 +14,7 @@ import { getWorkspaceViaEdge } from "@/lib/planetscale";
import { getLinkWithPartner } from "@/lib/planetscale/get-link-with-partner";
import { recordClick } from "@/lib/tinybird";
import { RedisLinkProps } from "@/lib/types";
-import { formatRedisLink, redis, redisUsEast } from "@/lib/upstash";
+import { formatRedisLink, redis, redisGlobal } from "@/lib/upstash";
import { DiscountSchema } from "@/lib/zod/schemas/discount";
import { PartnerSchema } from "@/lib/zod/schemas/partners";
import { isValidUrl, nanoid } from "@dub/utils";
@@ -61,7 +61,7 @@ export const POST = withAxiom(async (req) => {
const identityHash = await getIdentityHash(req);
const [cachedClickId, mgetResults] = await Promise.all([
- redisUsEast.get<string>(
+ redisGlobal.get<string>(
recordClickCache._createKey({ domain, key, identityHash }),
),
diff --git a/apps/web/app/(ee)/api/track/open/route.ts b/apps/web/app/(ee)/api/track/open/route.ts
index 2892f169ab..c8425f1c6d 100644
--- a/apps/web/app/(ee)/api/track/open/route.ts
+++ b/apps/web/app/(ee)/api/track/open/route.ts
@@ -9,7 +9,7 @@ import { getIdentityHash } from "@/lib/middleware/utils/get-identity-hash";
import { getLinkViaEdge } from "@/lib/planetscale";
import { recordClick } from "@/lib/tinybird";
import { RedisLinkProps } from "@/lib/types";
-import { formatRedisLink, redis, redisUsEast } from "@/lib/upstash";
+import { formatRedisLink, redis, redisGlobal } from "@/lib/upstash";
import {
trackOpenRequestSchema,
trackOpenResponseSchema,
@@ -71,7 +71,7 @@ export const POST = withAxiom(async (req) => {
const key = deepLink.pathname.slice(1) || "_root"; // Remove leading slash, default to _root if empty
let [cachedClickId, cachedLink] = await Promise.all([
- redisUsEast.get<string>(
+ redisGlobal.get<string>(
recordClickCache._createKey({ domain, key, identityHash }),
),
redis.get<RedisLinkProps>(linkCache._createKey({ domain, key })),
diff --git a/apps/web/app/(ee)/api/track/visit/route.ts b/apps/web/app/(ee)/api/track/visit/route.ts
index b724fd6a30..e221a58a2e 100644
--- a/apps/web/app/(ee)/api/track/visit/route.ts
+++ b/apps/web/app/(ee)/api/track/visit/route.ts
@@ -9,7 +9,7 @@ import { getIdentityHash } from "@/lib/middleware/utils/get-identity-hash";
import { getLinkViaEdge, getWorkspaceViaEdge } from "@/lib/planetscale";
import { recordClick } from "@/lib/tinybird";
import { RedisLinkProps } from "@/lib/types";
-import { formatRedisLink, redis, redisUsEast } from "@/lib/upstash";
+import { formatRedisLink, redis, redisGlobal } from "@/lib/upstash";
import { isValidUrl, nanoid } from "@dub/utils";
import { waitUntil } from "@vercel/functions";
import { NextResponse } from "next/server";
@@ -36,7 +36,7 @@ export const POST = withAxiom(async (req) => {
const identityHash = await getIdentityHash(req);
let [clickId, cachedLink] = await Promise.all([
- redisUsEast.get<string>(
+ redisGlobal.get<string>(
recordClickCache._createKey({ domain, key, identityHash }),
),
redis.get<RedisLinkProps>(linkCache._createKey({ domain, key })),
diff --git a/apps/web/lib/api/fraud/rules/check-customer-email-suspicious.ts b/apps/web/lib/api/fraud/rules/check-customer-email-suspicious.ts
index 53f73ce56c..38c330ad30 100644
--- a/apps/web/lib/api/fraud/rules/check-customer-email-suspicious.ts
+++ b/apps/web/lib/api/fraud/rules/check-customer-email-suspicious.ts
@@ -1,5 +1,5 @@
import { FraudEventContext } from "@/lib/types";
-import { redisWithTimeout } from "@/lib/upstash/redis";
+import { redis } from "@/lib/upstash/redis";
import { defineFraudRule } from "../define-fraud-rule";
export const checkCustomerEmailSuspicious = defineFraudRule({
@@ -23,7 +23,7 @@ export const checkCustomerEmailSuspicious = defineFraudRule({
const domain = emailParts[1].toLowerCase().trim();
try {
- const isDisposable = await redisWithTimeout.sismember(
+ const isDisposable = await redis.sismember(
"disposableEmailDomains",
domain,
);
diff --git a/apps/web/lib/api/links/cache.ts b/apps/web/lib/api/links/cache.ts
index c13872b58e..aefa3e5a9f 100644
--- a/apps/web/lib/api/links/cache.ts
+++ b/apps/web/lib/api/links/cache.ts
@@ -1,22 +1,20 @@
import { LinkProps, RedisLinkProps } from "@/lib/types";
-import { formatRedisLink, redis, redisWithTimeout } from "@/lib/upstash";
+import { formatRedisLink, redisGlobal } from "@/lib/upstash";
import { LRUCache } from "lru-cache";
import { decodeKey, isCaseSensitiveDomain } from "./case-sensitivity";
import { ExpandedLink } from "./utils/transform-link";
/*
* Link LRU cache to reduce Redis load during traffic spikes.
- * Max 5000 entries with 3-second TTL.
+ * Max 10,000 entries with 5-second TTL.
*/
const linkLRUCache = new LRUCache<string, RedisLinkProps>({
- max: 5000, // max 5000 entries
- ttl: 3000, // 3 seconds
+ max: 10000, // max 10,000 entries
+ ttl: 5000, // 5 seconds
});
/*
* Link cache expiration is set to 24 hours by default for all links.
- * Caveat: we don't set expiration for links with webhooks since it's expensive
- * to fetch and set on-demand inside link middleware.
*/
const CACHE_EXPIRATION = 60 * 60 * 24;
@@ -26,24 +24,12 @@ class LinkCache {
return;
}
- const pipeline = redis.pipeline();
+ const pipeline = redisGlobal.pipeline();
- const redisLinks = await Promise.all(
- links.map((link) => ({
- ...formatRedisLink(link),
- cacheKey: this._createKey({ domain: link.domain, key: link.key }),
- })),
- );
-
- redisLinks.map(({ cacheKey, ...redisLink }) => {
- const hasWebhooks =
- redisLink.webhookIds && redisLink.webhookIds.length > 0;
-
- pipeline.set(
- cacheKey,
- redisLink,
- hasWebhooks ? undefined : { ex: CACHE_EXPIRATION },
- );
+ links.forEach((link) => {
+ const redisLink = formatRedisLink(link);
+ const cacheKey = this._createKey({ domain: link.domain, key: link.key });
+ pipeline.set(cacheKey, redisLink, { ex: CACHE_EXPIRATION });
});
return await pipeline.exec();
@@ -51,17 +37,12 @@ class LinkCache {
async set(link: ExpandedLink) {
const redisLink = formatRedisLink(link);
- const hasWebhooks = redisLink.webhookIds && redisLink.webhookIds.length > 0;
const cacheKey = this._createKey({ domain: link.domain, key: link.key });
// Update LRU cache immediately to prevent stale reads
linkLRUCache.set(cacheKey, redisLink);
- return await redis.set(
- cacheKey,
- redisLink,
- hasWebhooks ? undefined : { ex: CACHE_EXPIRATION },
- );
+ return await redisGlobal.set(cacheKey, redisLink, { ex: CACHE_EXPIRATION });
}
async get({ domain, key }: Pick<LinkProps, "domain" | "key">) {
@@ -75,38 +56,28 @@ class LinkCache {
if (cachedLink) {
console.log(`[LRU Cache HIT] ${cacheKey}`);
+ linkLRUCache.set(cacheKey, cachedLink); // refresh the LRU cache
return cachedLink;
}
- console.log(`[LRU Cache MISS] ${cacheKey} - Checking Redis...`);
-
- try {
- // we're using the special redisWithTimeout client in case Redis times out
- cachedLink = await redisWithTimeout.get<RedisLinkProps>(cacheKey);
+ console.log(`[LRU Cache MISS] ${cacheKey} - Checking redisGlobal...`);
- if (cachedLink) {
- console.log(`[Redis Cache HIT] ${cacheKey} - Populating LRU Cache...`);
-
- linkLRUCache.set(cacheKey, cachedLink);
- } else {
- console.log(
- `[Redis Cache MISS] ${cacheKey} - Not found in LRU or Redis, falling back to MySQL...`,
- );
- }
+ // Fallback to redisGlobal if LRU cache miss
+ cachedLink = await redisGlobal.get<RedisLinkProps>(cacheKey);
+ if (cachedLink) {
+ console.log(`[Redis Cache HIT] ${cacheKey} - Populating LRU Cache...`);
+ linkLRUCache.set(cacheKey, cachedLink); // persist to LRU cache
return cachedLink;
- } catch (error) {
- console.error(
- "[LinkCache] – Timeout getting cached link from Redis, falling back to MySQL...",
- error,
+ } else {
+ console.log(
+ `[Redis Cache MISS] ${cacheKey} - Not found in LRU or Redis, falling back to MySQL...`,
);
-
- return null;
}
}
async delete({ domain, key }: Pick<LinkProps, "domain" | "key">) {
- return await redis.del(this._createKey({ domain, key }));
+ return await redisGlobal.del(this._createKey({ domain, key }));
}
async deleteMany(links: Pick<LinkProps, "domain" | "key">[]) {
@@ -114,7 +85,7 @@ class LinkCache {
return;
}
- const pipeline = redis.pipeline();
+ const pipeline = redisGlobal.pipeline();
links.forEach(({ domain, key }) => {
pipeline.del(this._createKey({ domain, key }));
@@ -128,7 +99,7 @@ class LinkCache {
return;
}
- const pipeline = redis.pipeline();
+ const pipeline = redisGlobal.pipeline();
links.forEach(({ domain, key }) => {
// expire the link cache key immediately
diff --git a/apps/web/lib/api/links/record-click-cache.ts b/apps/web/lib/api/links/record-click-cache.ts
index fb15b1b90f..3ef028df16 100644
--- a/apps/web/lib/api/links/record-click-cache.ts
+++ b/apps/web/lib/api/links/record-click-cache.ts
@@ -1,4 +1,4 @@
-import { redisUsEast } from "@/lib/upstash";
+import { redisGlobal } from "@/lib/upstash";
// Cache the click ID in Redis for 1 hour
const CACHE_EXPIRATION = 60 * 60;
@@ -16,7 +16,7 @@ class RecordClickCache {
identityHash,
clickId,
}: KeyProps & { clickId: string }) {
- return await redisUsEast.set(
+ return await redisGlobal.set(
this._createKey({ domain, key, identityHash }),
clickId,
{
@@ -26,7 +26,7 @@ class RecordClickCache {
}
async get({ domain, key, identityHash }: KeyProps) {
- return await redisUsEast.get<string>(
+ return await redisGlobal.get<string>(
this._createKey({ domain, key, identityHash }),
);
}
diff --git a/apps/web/lib/planetscale/get-link-via-edge.ts b/apps/web/lib/planetscale/get-link-via-edge.ts
index 3c04dada4a..015f904ad0 100644
--- a/apps/web/lib/planetscale/get-link-via-edge.ts
+++ b/apps/web/lib/planetscale/get-link-via-edge.ts
@@ -5,7 +5,7 @@ import {
isCaseSensitiveDomain,
} from "../api/links/case-sensitivity";
import { conn } from "./connection";
-import { EdgeLinkProps } from "./types";
+import { EdgeLinkProps, EdgeLinkWithWebhooks } from "./types";
export const getLinkViaEdge = async ({
domain,
@@ -13,7 +13,7 @@ export const getLinkViaEdge = async ({
}: {
domain: string;
key: string;
-}) => {
+}): Promise<EdgeLinkWithWebhooks | null> => {
const isCaseSensitive = isCaseSensitiveDomain(domain);
const keyToQuery = isCaseSensitive
? // for case sensitive domains, we need to encode the key
@@ -23,17 +23,26 @@ export const getLinkViaEdge = async ({
punyEncode(decodeURIComponent(key));
const { rows } =
- (await conn.execute("SELECT * FROM Link WHERE domain = ? AND `key` = ?", [
- domain,
- keyToQuery,
- ])) || {};
+ (await conn.execute(
+ `SELECT Link.*, LinkWebhook.webhookId
+ FROM Link
+ LEFT JOIN LinkWebhook ON Link.id = LinkWebhook.linkId
+ WHERE Link.domain = ? AND Link.\`key\` = ?`,
+ [domain, keyToQuery],
+ )) || {};
- const link =
- rows && Array.isArray(rows) && rows.length > 0
- ? (rows[0] as EdgeLinkProps)
- : null;
+ if (!rows || !Array.isArray(rows) || rows.length === 0) return null;
- return link
- ? { ...link, key: decodeKeyIfCaseSensitive({ domain, key }) }
- : null;
+ const first = rows[0] as EdgeLinkProps & { webhookId: string | null };
+ const { webhookId: _w, ...link } = first;
+ const webhooks = (rows as (EdgeLinkProps & { webhookId: string | null })[])
+ .map((r) => r.webhookId)
+ .filter((id): id is string => id != null)
+ .map((webhookId) => ({ webhookId }));
+
+ return {
+ ...link,
+ key: decodeKeyIfCaseSensitive({ domain, key }),
+ webhooks,
+ };
};
diff --git a/apps/web/lib/planetscale/types.ts b/apps/web/lib/planetscale/types.ts
index 3c1fff5329..4f667c2f36 100644
--- a/apps/web/lib/planetscale/types.ts
+++ b/apps/web/lib/planetscale/types.ts
@@ -24,6 +24,11 @@ export interface EdgeLinkProps {
partnerId: string | null;
}
+/** Link from edge (getLinkViaEdge) with webhooks, matching ExpandedLink shape */
+export type EdgeLinkWithWebhooks = EdgeLinkProps & {
+ webhooks: { webhookId: string }[];
+};
+
export interface EdgeDomainProps {
id: string;
slug: string;
diff --git a/apps/web/lib/upstash/redis.ts b/apps/web/lib/upstash/redis.ts
index 3d067baee6..40a9bf47c3 100644
--- a/apps/web/lib/upstash/redis.ts
+++ b/apps/web/lib/upstash/redis.ts
@@ -6,23 +6,43 @@ export const redis = new Redis({
token: process.env.UPSTASH_REDIS_REST_TOKEN || "",
});
-// Special Redis instance with timeout
-export const redisWithTimeout = new Redis({
- url: process.env.UPSTASH_REDIS_REST_URL || "",
- token: process.env.UPSTASH_REDIS_REST_TOKEN || "",
+// This is a separate global Redis instance that we use
+// for global operations (e.g. linkCache, recordClick)
+// so that if this redis goes down, it won't impact other endpoints
+const hasGlobalRedisConfig =
+ !!process.env.UPSTASH_GLOBAL_REDIS_REST_URL &&
+ !!process.env.UPSTASH_GLOBAL_REDIS_REST_TOKEN;
+
+const redisGlobalBase = new Redis({
+ url: hasGlobalRedisConfig
+ ? process.env.UPSTASH_GLOBAL_REDIS_REST_URL
+ : process.env.UPSTASH_REDIS_REST_URL || "",
+ token: hasGlobalRedisConfig
+ ? process.env.UPSTASH_GLOBAL_REDIS_REST_TOKEN
+ : process.env.UPSTASH_REDIS_REST_TOKEN || "",
signal: () => AbortSignal.timeout(1000),
});
-// This is a separate us-east-1 Redis instance that we use
-// for high-volume ratelimit operations (e.g. recordClick)
-// so that if this redis goes down, it won't impact other endpoints
-export const redisUsEast = new Redis({
- url:
- process.env.UPSTASH_US_EAST_REDIS_REST_URL ||
- process.env.UPSTASH_REDIS_REST_URL ||
- "",
- token:
- process.env.UPSTASH_US_EAST_REDIS_REST_TOKEN ||
- process.env.UPSTASH_REDIS_REST_TOKEN ||
- "",
+// On timeout (or other errors), get() method calls return null instead of throwing
+export const redisGlobal = new Proxy(redisGlobalBase, {
+ get(target, prop) {
+ const value = target[prop as keyof Redis];
+ if (prop === "get" && typeof value === "function") {
+ return async (...args: unknown[]) => {
+ try {
+ return await (value as (...args: unknown[]) => unknown).apply(
+ target,
+ args,
+ );
+ } catch (error) {
+ console.error(
+ "[redisGlobal] – Timeout getting value from Redis, returning null...",
+ error,
+ );
+ return null;
+ }
+ };
+ }
+ return value;
+ },
});