Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Improve notfound cache (#3638)

Steven Teyedc9a59b1eef
diff --git a/apps/web/app/[domain]/banned/page.tsx b/apps/web/app/[domain]/banned/page.tsx
index 09d2319915..9c67f90270 100644
--- a/apps/web/app/[domain]/banned/page.tsx
+++ b/apps/web/app/[domain]/banned/page.tsx
@@ -24,7 +24,7 @@ export function generateStaticParams() {
return [];
}
-export default async function BannedPage() {
+export default function BannedPage() {
return (
<div>
<Hero>
diff --git a/apps/web/app/[domain]/notfound/page.tsx b/apps/web/app/[domain]/notfound/page.tsx
index 6e82cd5306..3c5eb02a61 100644
--- a/apps/web/app/[domain]/notfound/page.tsx
+++ b/apps/web/app/[domain]/notfound/page.tsx
@@ -4,8 +4,10 @@ import { CTA } from "@/ui/placeholders/cta";
import { FeaturesSection } from "@/ui/placeholders/features-section";
import { Hero } from "@/ui/placeholders/hero";
import { LearnMoreButton } from "@/ui/placeholders/learn-more-button";
+import { prisma } from "@dub/prisma";
import { GlobeSearch } from "@dub/ui";
import { cn, constructMetadata } from "@dub/utils";
+import { redirect } from "next/navigation";
export const revalidate = false; // cache indefinitely
@@ -26,7 +28,20 @@ export function generateStaticParams() {
return [];
}
-export default async function NotFoundLinkPage() {
+export default async function NotFoundLinkPage(props: {
+ params: Promise<{ domain: string }>;
+}) {
+ const { domain } = await props.params;
+ const domainData = await prisma.domain.findUnique({
+ where: {
+ slug: domain,
+ },
+ });
+
+ if (domainData?.notFoundUrl) {
+ redirect(domainData.notFoundUrl);
+ }
+
return (
<main className="flex min-h-screen flex-col justify-between">
<Hero>
diff --git a/apps/web/app/api/domains/[domain]/route.ts b/apps/web/app/api/domains/[domain]/route.ts
index e85f966360..fe9e45d78a 100644
--- a/apps/web/app/api/domains/[domain]/route.ts
+++ b/apps/web/app/api/domains/[domain]/route.ts
@@ -16,7 +16,7 @@ import { prisma } from "@dub/prisma";
import { Prisma } from "@dub/prisma/client";
import { combineWords, nanoid, R2_URL } from "@dub/utils";
import { waitUntil } from "@vercel/functions";
-import { revalidateTag } from "next/cache";
+import { revalidatePath, revalidateTag } from "next/cache";
import { NextResponse } from "next/server";
import * as z from "zod/v4";
@@ -233,13 +233,22 @@ export const PATCH = withWorkspace(
]);
}
- // invalidate notfound cache
+ // invalidate static / isr cached for notfound links
if (
- (notFoundUrl !== undefined &&
- notFoundUrl !== existingDomain.notFoundUrl) ||
- (expiredUrl !== undefined && expiredUrl !== existingDomain.expiredUrl)
+ notFoundUrl !== undefined &&
+ notFoundUrl !== existingDomain.notFoundUrl
) {
- revalidateTag(`notfound:${domain.toLowerCase()}`);
+ revalidateTag(`static:${domain.toLowerCase()}`);
+ revalidatePath(`/${domain.toLowerCase()}/notfound`);
+ }
+
+ // invalidate static / isr cached for expired links
+ if (
+ expiredUrl !== undefined &&
+ expiredUrl !== existingDomain.expiredUrl
+ ) {
+ revalidateTag(`static:${domain.toLowerCase()}`);
+ revalidatePath(`/${domain.toLowerCase()}/expired`);
}
// invalidate wellknown cache if any of the wellknown files have changed
diff --git a/apps/web/lib/api/links/cache.ts b/apps/web/lib/api/links/cache.ts
index 081e483f55..cdb40a5a88 100644
--- a/apps/web/lib/api/links/cache.ts
+++ b/apps/web/lib/api/links/cache.ts
@@ -41,7 +41,7 @@ class LinkCache {
const redisLink = formatRedisLink(link);
const cacheKey = this._createKey({ domain: link.domain, key: link.key });
pipeline.set(cacheKey, redisLink, { ex: REDIS_CACHE_EXPIRATION });
- revalidateTag(`notfound:${link.domain.toLowerCase()}:${link.key}`);
+ revalidateTag(`static:${link.domain.toLowerCase()}:${link.key}`);
});
return await pipeline.exec();
@@ -53,7 +53,7 @@ class LinkCache {
// Update LRU cache immediately to prevent stale reads
linkLRUCache.set(cacheKey, redisLink);
- revalidateTag(`notfound:${link.domain.toLowerCase()}:${link.key}`);
+ revalidateTag(`static:${link.domain.toLowerCase()}:${link.key}`);
await Promise.all([
redisGlobal.set(cacheKey, redisLink, {
@@ -170,12 +170,15 @@ class LinkCache {
return caseSensitive ? cacheKey : cacheKey.toLowerCase();
}
- _createNotFoundCacheKeys({ domain, key }: Pick<LinkProps, "domain" | "key">) {
+ _createStaticPagesCacheKeys({
+ domain,
+ key,
+ }: Pick<LinkProps, "domain" | "key">) {
domain = domain.toLowerCase();
// here we set 2 cache tags to invalidate the cache:
- // 1. notfound:${domain}:${key} - for the specific not found link
- // 2. notfound:${domain} - scope all links under the domain (for easy purging in PATCH /domains/:domain)
- return `notfound:${domain}:${key},notfound:${domain}`;
+ // 1. static:${domain}:${key} - for the specific static page
+ // 2. static:${domain} - scope all links under the domain (for easy purging in PATCH /domains/:domain)
+ return `static:${domain}:${key},static:${domain}`;
}
// Vercel cache reads are 10x cheaper than writes, so to invalidate the cache
diff --git a/apps/web/lib/middleware/link.ts b/apps/web/lib/middleware/link.ts
index 0a2d9f4503..dde2f48d6e 100644
--- a/apps/web/lib/middleware/link.ts
+++ b/apps/web/lib/middleware/link.ts
@@ -29,7 +29,6 @@ import { createResponseWithCookies } from "./utils/create-response-with-cookies"
import { detectBot } from "./utils/detect-bot";
import { getFinalUrl } from "./utils/get-final-url";
import { getIdentityHash } from "./utils/get-identity-hash";
-import { handleNotFoundLink } from "./utils/handle-not-found-link";
import { isIosAppStoreUrl } from "./utils/is-ios-app-store-url";
import { isSingularTrackingUrl } from "./utils/is-singular-tracking-url";
import { isSupportedCustomURIScheme } from "./utils/is-supported-custom-uri-scheme";
@@ -62,6 +61,14 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {
key = "_root";
}
+ const STATIC_PAGES_CACHE_HEADERS = {
+ "Vercel-CDN-Cache-Control": "public, s-maxage=86400",
+ "Vercel-Cache-Tag": linkCache._createStaticPagesCacheKeys({
+ domain,
+ key: originalKey || "_root",
+ }),
+ };
+
// we don't support .php links (too much bot traffic)
// hence we redirect to the root domain and add `dub-no-track` header to avoid tracking bot traffic
if (isUnsupportedKey(key)) {
@@ -88,7 +95,12 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {
return await crawlBitly(req);
}
- return await handleNotFoundLink(req);
+ return NextResponse.rewrite(new URL(`/${domain}/notfound`, req.url), {
+ headers: {
+ ...DUB_HEADERS,
+ ...STATIC_PAGES_CACHE_HEADERS,
+ },
+ });
}
isPartnerLink = Boolean(linkData.programId && linkData.partnerId);
@@ -198,14 +210,20 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {
return NextResponse.rewrite(new URL(`/${domain}/banned`, req.url), {
headers: {
...DUB_HEADERS,
- ...(!shouldIndex && { "X-Robots-Tag": "googlebot: noindex" }),
+ ...STATIC_PAGES_CACHE_HEADERS,
+ "X-Robots-Tag": "googlebot: noindex",
},
});
}
// handle disabled links
if (disabledAt) {
- return await handleNotFoundLink(req);
+ return NextResponse.rewrite(new URL(`/${domain}/notfound`, req.url), {
+ headers: {
+ ...DUB_HEADERS,
+ ...STATIC_PAGES_CACHE_HEADERS,
+ },
+ });
}
// if the link has expired
@@ -222,6 +240,7 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {
return NextResponse.rewrite(new URL(`/${domain}/expired`, req.url), {
headers: {
...DUB_HEADERS,
+ ...STATIC_PAGES_CACHE_HEADERS,
...(!shouldIndex && { "X-Robots-Tag": "googlebot: noindex" }),
},
});
@@ -278,18 +297,11 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {
{
headers: {
...DUB_HEADERS,
+ ...STATIC_PAGES_CACHE_HEADERS,
...(!shouldIndex && { "X-Robots-Tag": "googlebot: noindex" }),
},
},
);
- rewriteResponse.headers.set(
- "Vercel-CDN-Cache-Control",
- "public, s-maxage=86400",
- );
- rewriteResponse.headers.set(
- "Vercel-Cache-Tag",
- linkCache._createNotFoundCacheKeys({ domain, key: "_root" }), // set cache tag for root domain link
- );
return createResponseWithCookies(rewriteResponse, cookieData);
}
diff --git a/apps/web/lib/middleware/utils/crawl-bitly.ts b/apps/web/lib/middleware/utils/crawl-bitly.ts
index f687f7ca07..aa218000cc 100644
--- a/apps/web/lib/middleware/utils/crawl-bitly.ts
+++ b/apps/web/lib/middleware/utils/crawl-bitly.ts
@@ -25,7 +25,7 @@ export const crawlBitly = async (req: NextRequest) => {
headers: {
...DUB_HEADERS,
"Vercel-CDN-Cache-Control": "public, s-maxage=86400",
- "Vercel-Cache-Tag": linkCache._createNotFoundCacheKeys({
+ "Vercel-Cache-Tag": linkCache._createStaticPagesCacheKeys({
domain,
key: fullKey,
}),
diff --git a/apps/web/lib/middleware/utils/handle-not-found-link.ts b/apps/web/lib/middleware/utils/handle-not-found-link.ts
deleted file mode 100644
index 12ddeb1952..0000000000
--- a/apps/web/lib/middleware/utils/handle-not-found-link.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-import { linkCache } from "@/lib/api/links/cache";
-import { getDomainViaEdge } from "@/lib/planetscale/get-domain-via-edge";
-import { DUB_HEADERS } from "@dub/utils";
-import { NextRequest, NextResponse } from "next/server";
-import { parse } from "./parse";
-
-export const handleNotFoundLink = async (req: NextRequest) => {
- const { domain, fullKey } = parse(req);
-
- let response: NextResponse;
- // check if domain has notFoundUrl configured
- const domainData = await getDomainViaEdge(domain);
- if (domainData?.notFoundUrl) {
- response = NextResponse.redirect(domainData.notFoundUrl, {
- headers: {
- ...DUB_HEADERS,
- "X-Robots-Tag": "googlebot: noindex",
- // pass the Referer [sic] value to the not found URL
- Referer: req.url,
- },
- status: 302,
- });
- } else {
- response = NextResponse.rewrite(new URL(`/${domain}/notfound`, req.url), {
- headers: DUB_HEADERS,
- });
- }
- response.headers.set("Vercel-CDN-Cache-Control", "public, s-maxage=86400");
- response.headers.set(
- "Vercel-Cache-Tag",
- linkCache._createNotFoundCacheKeys({ domain, key: fullKey }),
- );
- return response;
-};
diff --git a/apps/web/tests/redirects/index.test.ts b/apps/web/tests/redirects/index.test.ts
index 0fe6a5906c..94379fb06c 100644
--- a/apps/web/tests/redirects/index.test.ts
+++ b/apps/web/tests/redirects/index.test.ts
@@ -82,9 +82,12 @@ describe.runIf(env.CI)("Link Redirects", async () => {
test("disabled link", async () => {
const response = await fetch(`${h.baseUrl}/disabled`, fetchOptions);
- expect(response.headers.get("location")).toBe("https://dub.co/");
- expect(response.headers.get("x-powered-by")).toBe(poweredBy);
- expect(response.status).toBe(302);
+ // Special case for disabled/notfound links since we're using redirect() from next/navigation:
+ // - doesn't support x-powered-by header
+ // - uses 307 status code instead of 302
+ // This is the same as case-sensitive (incorrect) key test below.
+ expect(response.headers.get("location")).toBe("https://dub.co/links");
+ expect(response.status).toBe(307);
});
test("with slash", async () => {
@@ -281,9 +284,8 @@ describe.runIf(env.CI)("Link Redirects", async () => {
fetchOptions,
);
- expect(response.headers.get("location")).toBe("https://dub.co/");
- expect(response.headers.get("x-powered-by")).toBe(poweredBy);
- expect(response.status).toBe(302);
+ expect(response.headers.get("location")).toBe("https://dub.co/links");
+ expect(response.status).toBe(307);
});
test("with case-sensitive key (and dub_id)", async () => {