The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Improve notfound cache (#3638)
Steven Teyedc9a59b1eef
diff --git a/apps/web/app/[domain]/banned/page.tsx b/apps/web/app/[domain]/banned/page.tsxindex 09d2319915..9c67f90270 100644--- a/apps/web/app/[domain]/banned/page.tsx+++ b/apps/web/app/[domain]/banned/page.tsx@@ -24,7 +24,7 @@ export function generateStaticParams() {return [];}-export default async function BannedPage() {+export default function BannedPage() {return (<div><Hero>diff --git a/apps/web/app/[domain]/notfound/page.tsx b/apps/web/app/[domain]/notfound/page.tsxindex 6e82cd5306..3c5eb02a61 100644--- a/apps/web/app/[domain]/notfound/page.tsx+++ b/apps/web/app/[domain]/notfound/page.tsx@@ -4,8 +4,10 @@ import { CTA } from "@/ui/placeholders/cta";import { FeaturesSection } from "@/ui/placeholders/features-section";import { Hero } from "@/ui/placeholders/hero";import { LearnMoreButton } from "@/ui/placeholders/learn-more-button";+import { prisma } from "@dub/prisma";import { GlobeSearch } from "@dub/ui";import { cn, constructMetadata } from "@dub/utils";+import { redirect } from "next/navigation";export const revalidate = false; // cache indefinitely@@ -26,7 +28,20 @@ export function generateStaticParams() {return [];}-export default async function NotFoundLinkPage() {+export default async function NotFoundLinkPage(props: {+ params: Promise<{ domain: string }>;+}) {+ const { domain } = await props.params;+ const domainData = await prisma.domain.findUnique({+ where: {+ slug: domain,+ },+ });++ if (domainData?.notFoundUrl) {+ redirect(domainData.notFoundUrl);+ }+return (<main className="flex min-h-screen flex-col justify-between"><Hero>diff --git a/apps/web/app/api/domains/[domain]/route.ts b/apps/web/app/api/domains/[domain]/route.tsindex e85f966360..fe9e45d78a 100644--- a/apps/web/app/api/domains/[domain]/route.ts+++ b/apps/web/app/api/domains/[domain]/route.ts@@ -16,7 +16,7 @@ import { prisma } from "@dub/prisma";import { Prisma } from "@dub/prisma/client";import { combineWords, nanoid, R2_URL } from "@dub/utils";import { waitUntil } from "@vercel/functions";-import { revalidateTag } from "next/cache";+import { revalidatePath, revalidateTag } from "next/cache";import { NextResponse } from "next/server";import * as z from "zod/v4";@@ -233,13 +233,22 @@ export const PATCH = withWorkspace(]);}- // invalidate notfound cache+ // invalidate static / isr cached for notfound linksif (- (notFoundUrl !== undefined &&- notFoundUrl !== existingDomain.notFoundUrl) ||- (expiredUrl !== undefined && expiredUrl !== existingDomain.expiredUrl)+ notFoundUrl !== undefined &&+ notFoundUrl !== existingDomain.notFoundUrl) {- revalidateTag(`notfound:${domain.toLowerCase()}`);+ revalidateTag(`static:${domain.toLowerCase()}`);+ revalidatePath(`/${domain.toLowerCase()}/notfound`);+ }++ // invalidate static / isr cached for expired links+ if (+ expiredUrl !== undefined &&+ expiredUrl !== existingDomain.expiredUrl+ ) {+ revalidateTag(`static:${domain.toLowerCase()}`);+ revalidatePath(`/${domain.toLowerCase()}/expired`);}// invalidate wellknown cache if any of the wellknown files have changeddiff --git a/apps/web/lib/api/links/cache.ts b/apps/web/lib/api/links/cache.tsindex 081e483f55..cdb40a5a88 100644--- a/apps/web/lib/api/links/cache.ts+++ b/apps/web/lib/api/links/cache.ts@@ -41,7 +41,7 @@ class LinkCache {const redisLink = formatRedisLink(link);const cacheKey = this._createKey({ domain: link.domain, key: link.key });pipeline.set(cacheKey, redisLink, { ex: REDIS_CACHE_EXPIRATION });- revalidateTag(`notfound:${link.domain.toLowerCase()}:${link.key}`);+ revalidateTag(`static:${link.domain.toLowerCase()}:${link.key}`);});return await pipeline.exec();@@ -53,7 +53,7 @@ class LinkCache {// Update LRU cache immediately to prevent stale readslinkLRUCache.set(cacheKey, redisLink);- revalidateTag(`notfound:${link.domain.toLowerCase()}:${link.key}`);+ revalidateTag(`static:${link.domain.toLowerCase()}:${link.key}`);await Promise.all([redisGlobal.set(cacheKey, redisLink, {@@ -170,12 +170,15 @@ class LinkCache {return caseSensitive ? cacheKey : cacheKey.toLowerCase();}- _createNotFoundCacheKeys({ domain, key }: Pick<LinkProps, "domain" | "key">) {+ _createStaticPagesCacheKeys({+ domain,+ key,+ }: Pick<LinkProps, "domain" | "key">) {domain = domain.toLowerCase();// here we set 2 cache tags to invalidate the cache:- // 1. notfound:${domain}:${key} - for the specific not found link- // 2. notfound:${domain} - scope all links under the domain (for easy purging in PATCH /domains/:domain)- return `notfound:${domain}:${key},notfound:${domain}`;+ // 1. static:${domain}:${key} - for the specific static page+ // 2. static:${domain} - scope all links under the domain (for easy purging in PATCH /domains/:domain)+ return `static:${domain}:${key},static:${domain}`;}// Vercel cache reads are 10x cheaper than writes, so to invalidate the cachediff --git a/apps/web/lib/middleware/link.ts b/apps/web/lib/middleware/link.tsindex 0a2d9f4503..dde2f48d6e 100644--- a/apps/web/lib/middleware/link.ts+++ b/apps/web/lib/middleware/link.ts@@ -29,7 +29,6 @@ import { createResponseWithCookies } from "./utils/create-response-with-cookies"import { detectBot } from "./utils/detect-bot";import { getFinalUrl } from "./utils/get-final-url";import { getIdentityHash } from "./utils/get-identity-hash";-import { handleNotFoundLink } from "./utils/handle-not-found-link";import { isIosAppStoreUrl } from "./utils/is-ios-app-store-url";import { isSingularTrackingUrl } from "./utils/is-singular-tracking-url";import { isSupportedCustomURIScheme } from "./utils/is-supported-custom-uri-scheme";@@ -62,6 +61,14 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {key = "_root";}+ const STATIC_PAGES_CACHE_HEADERS = {+ "Vercel-CDN-Cache-Control": "public, s-maxage=86400",+ "Vercel-Cache-Tag": linkCache._createStaticPagesCacheKeys({+ domain,+ key: originalKey || "_root",+ }),+ };+// we don't support .php links (too much bot traffic)// hence we redirect to the root domain and add `dub-no-track` header to avoid tracking bot trafficif (isUnsupportedKey(key)) {@@ -88,7 +95,12 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {return await crawlBitly(req);}- return await handleNotFoundLink(req);+ return NextResponse.rewrite(new URL(`/${domain}/notfound`, req.url), {+ headers: {+ ...DUB_HEADERS,+ ...STATIC_PAGES_CACHE_HEADERS,+ },+ });}isPartnerLink = Boolean(linkData.programId && linkData.partnerId);@@ -198,14 +210,20 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {return NextResponse.rewrite(new URL(`/${domain}/banned`, req.url), {headers: {...DUB_HEADERS,- ...(!shouldIndex && { "X-Robots-Tag": "googlebot: noindex" }),+ ...STATIC_PAGES_CACHE_HEADERS,+ "X-Robots-Tag": "googlebot: noindex",},});}// handle disabled linksif (disabledAt) {- return await handleNotFoundLink(req);+ return NextResponse.rewrite(new URL(`/${domain}/notfound`, req.url), {+ headers: {+ ...DUB_HEADERS,+ ...STATIC_PAGES_CACHE_HEADERS,+ },+ });}// if the link has expired@@ -222,6 +240,7 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {return NextResponse.rewrite(new URL(`/${domain}/expired`, req.url), {headers: {...DUB_HEADERS,+ ...STATIC_PAGES_CACHE_HEADERS,...(!shouldIndex && { "X-Robots-Tag": "googlebot: noindex" }),},});@@ -278,18 +297,11 @@ export async function LinkMiddleware(req: NextRequest, ev: NextFetchEvent) {{headers: {...DUB_HEADERS,+ ...STATIC_PAGES_CACHE_HEADERS,...(!shouldIndex && { "X-Robots-Tag": "googlebot: noindex" }),},},);- rewriteResponse.headers.set(- "Vercel-CDN-Cache-Control",- "public, s-maxage=86400",- );- rewriteResponse.headers.set(- "Vercel-Cache-Tag",- linkCache._createNotFoundCacheKeys({ domain, key: "_root" }), // set cache tag for root domain link- );return createResponseWithCookies(rewriteResponse, cookieData);}diff --git a/apps/web/lib/middleware/utils/crawl-bitly.ts b/apps/web/lib/middleware/utils/crawl-bitly.tsindex f687f7ca07..aa218000cc 100644--- a/apps/web/lib/middleware/utils/crawl-bitly.ts+++ b/apps/web/lib/middleware/utils/crawl-bitly.ts@@ -25,7 +25,7 @@ export const crawlBitly = async (req: NextRequest) => {headers: {...DUB_HEADERS,"Vercel-CDN-Cache-Control": "public, s-maxage=86400",- "Vercel-Cache-Tag": linkCache._createNotFoundCacheKeys({+ "Vercel-Cache-Tag": linkCache._createStaticPagesCacheKeys({domain,key: fullKey,}),diff --git a/apps/web/lib/middleware/utils/handle-not-found-link.ts b/apps/web/lib/middleware/utils/handle-not-found-link.tsdeleted file mode 100644index 12ddeb1952..0000000000--- a/apps/web/lib/middleware/utils/handle-not-found-link.ts+++ /dev/null@@ -1,34 +0,0 @@-import { linkCache } from "@/lib/api/links/cache";-import { getDomainViaEdge } from "@/lib/planetscale/get-domain-via-edge";-import { DUB_HEADERS } from "@dub/utils";-import { NextRequest, NextResponse } from "next/server";-import { parse } from "./parse";--export const handleNotFoundLink = async (req: NextRequest) => {- const { domain, fullKey } = parse(req);-- let response: NextResponse;- // check if domain has notFoundUrl configured- const domainData = await getDomainViaEdge(domain);- if (domainData?.notFoundUrl) {- response = NextResponse.redirect(domainData.notFoundUrl, {- headers: {- ...DUB_HEADERS,- "X-Robots-Tag": "googlebot: noindex",- // pass the Referer [sic] value to the not found URL- Referer: req.url,- },- status: 302,- });- } else {- response = NextResponse.rewrite(new URL(`/${domain}/notfound`, req.url), {- headers: DUB_HEADERS,- });- }- response.headers.set("Vercel-CDN-Cache-Control", "public, s-maxage=86400");- response.headers.set(- "Vercel-Cache-Tag",- linkCache._createNotFoundCacheKeys({ domain, key: fullKey }),- );- return response;-};diff --git a/apps/web/tests/redirects/index.test.ts b/apps/web/tests/redirects/index.test.tsindex 0fe6a5906c..94379fb06c 100644--- a/apps/web/tests/redirects/index.test.ts+++ b/apps/web/tests/redirects/index.test.ts@@ -82,9 +82,12 @@ describe.runIf(env.CI)("Link Redirects", async () => {test("disabled link", async () => {const response = await fetch(`${h.baseUrl}/disabled`, fetchOptions);- expect(response.headers.get("location")).toBe("https://dub.co/");- expect(response.headers.get("x-powered-by")).toBe(poweredBy);- expect(response.status).toBe(302);+ // Special case for disabled/notfound links since we're using redirect() from next/navigation:+ // - doesn't support x-powered-by header+ // - uses 307 status code instead of 302+ // This is the same as case-sensitive (incorrect) key test below.+ expect(response.headers.get("location")).toBe("https://dub.co/links");+ expect(response.status).toBe(307);});test("with slash", async () => {@@ -281,9 +284,8 @@ describe.runIf(env.CI)("Link Redirects", async () => {fetchOptions,);- expect(response.headers.get("location")).toBe("https://dub.co/");- expect(response.headers.get("x-powered-by")).toBe(poweredBy);- expect(response.status).toBe(302);+ expect(response.headers.get("location")).toBe("https://dub.co/links");+ expect(response.status).toBe(307);});test("with case-sensitive key (and dub_id)", async () => {