Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

apply safe fetch in more places

Pedro Ladeira4b958aa4929f
diff --git a/apps/web/app/api/domains/[domain]/validate/route.ts b/apps/web/app/api/domains/[domain]/validate/route.ts
index 629706460e..1904d0741f 100644
--- a/apps/web/app/api/domains/[domain]/validate/route.ts
+++ b/apps/web/app/api/domains/[domain]/validate/route.ts
@@ -1,6 +1,7 @@
import { isValidDomain } from "@/lib/api/domains/is-valid-domain";
import { domainExists } from "@/lib/api/domains/utils";
import { validateDubLinkSubdomain } from "@/lib/api/domains/validate-dub-link-subdomain";
+import { safeFetch } from "@/lib/api/safe-fetch";
import { withSession } from "@/lib/auth";
import dns from "dns/promises";
import { NextResponse } from "next/server";
@@ -50,25 +51,21 @@ export const GET = withSession(async ({ params }) => {
// Helper function to check if a site is active on the domain
async function hasSiteConfigured(domain: string): Promise<boolean> {
try {
- // Try HTTP HEAD request first (both HTTP and HTTPS)
+ // Try HTTP HEAD request first (both HTTP and HTTPS).
+ // safeFetch enforces SSRF guards on the user-supplied domain (the path
+ // param flows in here) so this can't be used to probe internal hosts.
const urls = [`https://${domain}`, `http://${domain}`];
for (const url of urls) {
try {
- const controller = new AbortController();
- const timeoutId = setTimeout(() => controller.abort(), 3000); // 3 second timeout
- const response = await fetch(url, {
- method: "HEAD",
- signal: controller.signal,
- });
- clearTimeout(timeoutId);
+ const response = await safeFetch(
+ url,
+ { method: "HEAD" },
+ { timeoutMs: 3000 },
+ );
if (response.ok) return true;
- } catch (e) {
- if (e instanceof DOMException && e.name === "AbortError") {
- // If request was aborted due to timeout, continue to next check
- continue;
- }
- // Continue to next URL if this one fails
+ } catch {
+ // Continue to next URL if this one fails (timeout, SSRF guard, etc.)
continue;
}
}
diff --git a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts b/apps/web/lib/sitemaps/import-tracked-sitemaps.ts
index 15256e684c..98e9b26bb9 100644
--- a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts
+++ b/apps/web/lib/sitemaps/import-tracked-sitemaps.ts
@@ -1,8 +1,8 @@
import { bulkCreateLinks } from "@/lib/api/links/bulk-create-links";
+import { safeFetch } from "@/lib/api/safe-fetch";
import type { TrackedSitemap } from "@/lib/sitemaps/site-visit-tracking";
import { ProcessedLinkProps } from "@/lib/types";
import { prisma } from "@dub/prisma";
-import { fetchWithTimeout } from "@dub/utils/src";
import { XMLParser } from "fast-xml-parser";
type SitemapXmlUrlEntry = {
@@ -72,7 +72,7 @@ async function decompressIfGzip(buffer: ArrayBuffer): Promise<string> {
async function fetchAndParseSitemap(
sitemapUrl: string,
): Promise<SitemapXmlResult> {
- const response = await fetchWithTimeout(sitemapUrl, { redirect: "error" }); // don't follow redirects
+ const response = await safeFetch(sitemapUrl, undefined, { maxRedirects: 0 });
const MAX_SITEMAP_BYTES = 10 * 1024 * 1024; // 10 MB
const contentLength = response.headers.get("content-length");
if (contentLength && parseInt(contentLength, 10) > MAX_SITEMAP_BYTES) {
diff --git a/apps/web/tests/misc/import-tracked-sitemaps.test.ts b/apps/web/tests/misc/import-tracked-sitemaps.test.ts
index 492e7e937d..0f920b059e 100644
--- a/apps/web/tests/misc/import-tracked-sitemaps.test.ts
+++ b/apps/web/tests/misc/import-tracked-sitemaps.test.ts
@@ -6,6 +6,8 @@ import { parseTrackedSitemaps } from "@/lib/sitemaps/site-visit-tracking";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { gzipSync } from "zlib";
+vi.mock("server-only", () => ({}));
+
function toArrayBuffer(buf: Buffer): ArrayBuffer {
const ab = new ArrayBuffer(buf.length);
new Uint8Array(ab).set(buf);
@@ -22,11 +24,14 @@ function makeFetchResponse(
status,
headers: new Headers(headers),
arrayBuffer: () => Promise.resolve(toArrayBuffer(body)),
+ json: () => Promise.resolve(JSON.parse(body.toString("utf8"))),
} as unknown as Response;
}
-function redirectResponse(location: string, status = 302): Response {
- return makeFetchResponse(Buffer.alloc(0), status, { Location: location });
+function dohResponseFor(ip: string, type: 1 | 28 = 1): Response {
+ return makeFetchResponse(
+ Buffer.from(JSON.stringify({ Answer: [{ data: ip, type }] })),
+ );
}
const urlsetXml = (urls: string[]) =>
@@ -134,7 +139,15 @@ describe("parseTrackedSitemaps", () => {
});
describe("crawlSitemapUrls", () => {
- const mockFetch = vi.fn();
+ const mockContentFetch = vi.fn();
+ const mockFetch = vi.fn((url: string | URL | Request) => {
+ const href = typeof url === "string" ? url : url.toString();
+ if (href.includes("dns.google/resolve")) {
+ // Return a public IP so safeFetch lets the request through.
+ return Promise.resolve(dohResponseFor("93.184.216.34"));
+ }
+ return mockContentFetch(href);
+ });
beforeEach(() => {
vi.stubGlobal("fetch", mockFetch);
@@ -151,7 +164,7 @@ describe("crawlSitemapUrls", () => {
"https://example.com/page-1",
"https://example.com/page-2",
];
- mockFetch.mockResolvedValue(
+ mockContentFetch.mockResolvedValue(
makeFetchResponse(Buffer.from(urlsetXml(pageUrls))),
);
@@ -165,7 +178,7 @@ describe("crawlSitemapUrls", () => {
});
it("deduplicates URLs across multiple entries", async () => {
- mockFetch.mockResolvedValue(
+ mockContentFetch.mockResolvedValue(
makeFetchResponse(
Buffer.from(
urlsetXml([
@@ -186,7 +199,7 @@ describe("crawlSitemapUrls", () => {
describe("sitemapindex", () => {
it("does not fetch nested sitemaps from a sitemap index", async () => {
- mockFetch.mockResolvedValueOnce(
+ mockContentFetch.mockResolvedValueOnce(
makeFetchResponse(
Buffer.from(
sitemapindexXml([
@@ -203,11 +216,11 @@ describe("crawlSitemapUrls", () => {
expect(urls).toEqual([]);
expect(hadErrors).toBe(false);
- expect(mockFetch).toHaveBeenCalledTimes(1);
+ expect(mockContentFetch).toHaveBeenCalledTimes(1);
});
it("returns no page URLs when the index only references itself", async () => {
- mockFetch.mockResolvedValue(
+ mockContentFetch.mockResolvedValue(
makeFetchResponse(
Buffer.from(sitemapindexXml(["https://example.com/sitemap.xml"])),
),
@@ -218,13 +231,13 @@ describe("crawlSitemapUrls", () => {
);
expect(urls).toEqual([]);
- expect(mockFetch).toHaveBeenCalledTimes(1);
+ expect(mockContentFetch).toHaveBeenCalledTimes(1);
});
});
describe("nested xml in urlset", () => {
it("skips .xml loc entries instead of treating them as pages", async () => {
- mockFetch.mockResolvedValue(
+ mockContentFetch.mockResolvedValue(
makeFetchResponse(
Buffer.from(
urlsetXml([
@@ -248,7 +261,7 @@ describe("crawlSitemapUrls", () => {
const many = Array.from({ length: MAX_URLS_PER_SITEMAP + 50 }, (_, i) =>
i === 0 ? "https://example.com/first" : `https://example.com/p/${i}`,
);
- mockFetch.mockResolvedValue(
+ mockContentFetch.mockResolvedValue(
makeFetchResponse(Buffer.from(urlsetXml(many))),
);
@@ -264,7 +277,7 @@ describe("crawlSitemapUrls", () => {
it("decompresses a gzip-compressed sitemap response", async () => {
const xml = urlsetXml(["https://example.com/page-1"]);
const compressed = gzipSync(xml);
- mockFetch.mockResolvedValue(makeFetchResponse(compressed));
+ mockContentFetch.mockResolvedValue(makeFetchResponse(compressed));
const { urls } = await crawlSitemapUrls(
"https://example.com/sitemap.xml.gz",
@@ -275,7 +288,7 @@ describe("crawlSitemapUrls", () => {
it("handles a plain XML response without attempting decompression", async () => {
const xml = urlsetXml(["https://example.com/page-1"]);
- mockFetch.mockResolvedValue(makeFetchResponse(Buffer.from(xml)));
+ mockContentFetch.mockResolvedValue(makeFetchResponse(Buffer.from(xml)));
const { urls } = await crawlSitemapUrls(
"https://example.com/sitemap.xml",