The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
apply safe fetch in more places
Pedro Ladeira4b958aa4929f
diff --git a/apps/web/app/api/domains/[domain]/validate/route.ts b/apps/web/app/api/domains/[domain]/validate/route.tsindex 629706460e..1904d0741f 100644--- a/apps/web/app/api/domains/[domain]/validate/route.ts+++ b/apps/web/app/api/domains/[domain]/validate/route.ts@@ -1,6 +1,7 @@import { isValidDomain } from "@/lib/api/domains/is-valid-domain";import { domainExists } from "@/lib/api/domains/utils";import { validateDubLinkSubdomain } from "@/lib/api/domains/validate-dub-link-subdomain";+import { safeFetch } from "@/lib/api/safe-fetch";import { withSession } from "@/lib/auth";import dns from "dns/promises";import { NextResponse } from "next/server";@@ -50,25 +51,21 @@ export const GET = withSession(async ({ params }) => {// Helper function to check if a site is active on the domainasync function hasSiteConfigured(domain: string): Promise<boolean> {try {- // Try HTTP HEAD request first (both HTTP and HTTPS)+ // Try HTTP HEAD request first (both HTTP and HTTPS).+ // safeFetch enforces SSRF guards on the user-supplied domain (the path+ // param flows in here) so this can't be used to probe internal hosts.const urls = [`https://${domain}`, `http://${domain}`];for (const url of urls) {try {- const controller = new AbortController();- const timeoutId = setTimeout(() => controller.abort(), 3000); // 3 second timeout- const response = await fetch(url, {- method: "HEAD",- signal: controller.signal,- });- clearTimeout(timeoutId);+ const response = await safeFetch(+ url,+ { method: "HEAD" },+ { timeoutMs: 3000 },+ );if (response.ok) return true;- } catch (e) {- if (e instanceof DOMException && e.name === "AbortError") {- // If request was aborted due to timeout, continue to next check- continue;- }- // Continue to next URL if this one fails+ } catch {+ // Continue to next URL if this one fails (timeout, SSRF guard, etc.)continue;}}diff --git a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts b/apps/web/lib/sitemaps/import-tracked-sitemaps.tsindex 15256e684c..98e9b26bb9 100644--- a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts+++ b/apps/web/lib/sitemaps/import-tracked-sitemaps.ts@@ -1,8 +1,8 @@import { bulkCreateLinks } from "@/lib/api/links/bulk-create-links";+import { safeFetch } from "@/lib/api/safe-fetch";import type { TrackedSitemap } from "@/lib/sitemaps/site-visit-tracking";import { ProcessedLinkProps } from "@/lib/types";import { prisma } from "@dub/prisma";-import { fetchWithTimeout } from "@dub/utils/src";import { XMLParser } from "fast-xml-parser";type SitemapXmlUrlEntry = {@@ -72,7 +72,7 @@ async function decompressIfGzip(buffer: ArrayBuffer): Promise<string> {async function fetchAndParseSitemap(sitemapUrl: string,): Promise<SitemapXmlResult> {- const response = await fetchWithTimeout(sitemapUrl, { redirect: "error" }); // don't follow redirects+ const response = await safeFetch(sitemapUrl, undefined, { maxRedirects: 0 });const MAX_SITEMAP_BYTES = 10 * 1024 * 1024; // 10 MBconst contentLength = response.headers.get("content-length");if (contentLength && parseInt(contentLength, 10) > MAX_SITEMAP_BYTES) {diff --git a/apps/web/tests/misc/import-tracked-sitemaps.test.ts b/apps/web/tests/misc/import-tracked-sitemaps.test.tsindex 492e7e937d..0f920b059e 100644--- a/apps/web/tests/misc/import-tracked-sitemaps.test.ts+++ b/apps/web/tests/misc/import-tracked-sitemaps.test.ts@@ -6,6 +6,8 @@ import { parseTrackedSitemaps } from "@/lib/sitemaps/site-visit-tracking";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { gzipSync } from "zlib";+vi.mock("server-only", () => ({}));+function toArrayBuffer(buf: Buffer): ArrayBuffer {const ab = new ArrayBuffer(buf.length);new Uint8Array(ab).set(buf);@@ -22,11 +24,14 @@ function makeFetchResponse(status,headers: new Headers(headers),arrayBuffer: () => Promise.resolve(toArrayBuffer(body)),+ json: () => Promise.resolve(JSON.parse(body.toString("utf8"))),} as unknown as Response;}-function redirectResponse(location: string, status = 302): Response {- return makeFetchResponse(Buffer.alloc(0), status, { Location: location });+function dohResponseFor(ip: string, type: 1 | 28 = 1): Response {+ return makeFetchResponse(+ Buffer.from(JSON.stringify({ Answer: [{ data: ip, type }] })),+ );}const urlsetXml = (urls: string[]) =>@@ -134,7 +139,15 @@ describe("parseTrackedSitemaps", () => {});describe("crawlSitemapUrls", () => {- const mockFetch = vi.fn();+ const mockContentFetch = vi.fn();+ const mockFetch = vi.fn((url: string | URL | Request) => {+ const href = typeof url === "string" ? url : url.toString();+ if (href.includes("dns.google/resolve")) {+ // Return a public IP so safeFetch lets the request through.+ return Promise.resolve(dohResponseFor("93.184.216.34"));+ }+ return mockContentFetch(href);+ });beforeEach(() => {vi.stubGlobal("fetch", mockFetch);@@ -151,7 +164,7 @@ describe("crawlSitemapUrls", () => {"https://example.com/page-1","https://example.com/page-2",];- mockFetch.mockResolvedValue(+ mockContentFetch.mockResolvedValue(makeFetchResponse(Buffer.from(urlsetXml(pageUrls))),);@@ -165,7 +178,7 @@ describe("crawlSitemapUrls", () => {});it("deduplicates URLs across multiple entries", async () => {- mockFetch.mockResolvedValue(+ mockContentFetch.mockResolvedValue(makeFetchResponse(Buffer.from(urlsetXml([@@ -186,7 +199,7 @@ describe("crawlSitemapUrls", () => {describe("sitemapindex", () => {it("does not fetch nested sitemaps from a sitemap index", async () => {- mockFetch.mockResolvedValueOnce(+ mockContentFetch.mockResolvedValueOnce(makeFetchResponse(Buffer.from(sitemapindexXml([@@ -203,11 +216,11 @@ describe("crawlSitemapUrls", () => {expect(urls).toEqual([]);expect(hadErrors).toBe(false);- expect(mockFetch).toHaveBeenCalledTimes(1);+ expect(mockContentFetch).toHaveBeenCalledTimes(1);});it("returns no page URLs when the index only references itself", async () => {- mockFetch.mockResolvedValue(+ mockContentFetch.mockResolvedValue(makeFetchResponse(Buffer.from(sitemapindexXml(["https://example.com/sitemap.xml"])),),@@ -218,13 +231,13 @@ describe("crawlSitemapUrls", () => {);expect(urls).toEqual([]);- expect(mockFetch).toHaveBeenCalledTimes(1);+ expect(mockContentFetch).toHaveBeenCalledTimes(1);});});describe("nested xml in urlset", () => {it("skips .xml loc entries instead of treating them as pages", async () => {- mockFetch.mockResolvedValue(+ mockContentFetch.mockResolvedValue(makeFetchResponse(Buffer.from(urlsetXml([@@ -248,7 +261,7 @@ describe("crawlSitemapUrls", () => {const many = Array.from({ length: MAX_URLS_PER_SITEMAP + 50 }, (_, i) =>i === 0 ? "https://example.com/first" : `https://example.com/p/${i}`,);- mockFetch.mockResolvedValue(+ mockContentFetch.mockResolvedValue(makeFetchResponse(Buffer.from(urlsetXml(many))),);@@ -264,7 +277,7 @@ describe("crawlSitemapUrls", () => {it("decompresses a gzip-compressed sitemap response", async () => {const xml = urlsetXml(["https://example.com/page-1"]);const compressed = gzipSync(xml);- mockFetch.mockResolvedValue(makeFetchResponse(compressed));+ mockContentFetch.mockResolvedValue(makeFetchResponse(compressed));const { urls } = await crawlSitemapUrls("https://example.com/sitemap.xml.gz",@@ -275,7 +288,7 @@ describe("crawlSitemapUrls", () => {it("handles a plain XML response without attempting decompression", async () => {const xml = urlsetXml(["https://example.com/page-1"]);- mockFetch.mockResolvedValue(makeFetchResponse(Buffer.from(xml)));+ mockContentFetch.mockResolvedValue(makeFetchResponse(Buffer.from(xml)));const { urls } = await crawlSitemapUrls("https://example.com/sitemap.xml",