Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Keep the first 2 MB of a large page in getHtml

Rafi09b262a0768e
diff --git a/apps/web/app/api/links/metatags/utils.ts b/apps/web/app/api/links/metatags/utils.ts
index 01d2de2e91..860e94651c 100644
--- a/apps/web/app/api/links/metatags/utils.ts
+++ b/apps/web/app/api/links/metatags/utils.ts
@@ -9,12 +9,12 @@ import { waitUntil } from "@vercel/functions";
import he from "he";
import { parse } from "node-html-parser";
-// Meta tags are in the <head>, which ends before 1 MB on real pages. Some full
-// pages (long Wikipedia articles) are near 3 MB, so the cap leaves room for them.
-export const MAX_HTML_BYTES = 5 * 1024 * 1024; // 5 MB
+// Meta tags are in the <head>, which ends before 1 MB on real pages (the largest
+// head we checked was about 760 KB). Only this much of a page is read.
+export const MAX_HTML_BYTES = 2 * 1024 * 1024; // 2 MB
-// Reads the body as UTF-8 text like response.text(), but stops and returns
-// null when the body passes maxBytes.
+// Reads the body as UTF-8 text like response.text(), but reads at most maxBytes.
+// A longer body is cut at maxBytes, because the meta tags are near the start.
const readTextWithLimit = async (response: Response, maxBytes: number) => {
if (!response.body) {
return "";
@@ -31,13 +31,17 @@ const readTextWithLimit = async (response: Response, maxBytes: number) => {
break;
}
- bytesRead += value.byteLength;
- if (bytesRead > maxBytes) {
+ const remaining = maxBytes - bytesRead;
+ const bytes =
+ value.byteLength > remaining ? value.subarray(0, remaining) : value;
+ bytesRead += bytes.byteLength;
+
+ if (bytesRead >= maxBytes) {
await reader.cancel().catch(() => {});
- return null;
+ return text + decoder.decode(bytes);
}
- text += decoder.decode(value, { stream: true });
+ text += decoder.decode(bytes, { stream: true });
}
return text + decoder.decode();
@@ -58,23 +62,8 @@ export const getHtml = async (url: string) => {
return null;
}
- const contentLength = Number(response.headers.get("content-length"));
- if (contentLength > MAX_HTML_BYTES) {
- await response.body?.cancel().catch(() => {});
- console.warn(
- `HTML response too large (Content-Length: ${contentLength}) for URL: ${url}`,
- );
- return null;
- }
-
- // Content-Length can be missing or wrong, so the stream is capped as well.
+ // Content-Length can be missing or wrong, so the limit is on the stream.
const text = await readTextWithLimit(response, MAX_HTML_BYTES);
- if (text === null) {
- console.warn(
- `HTML response passed ${MAX_HTML_BYTES} bytes for URL: ${url}`,
- );
- return null;
- }
// Check if the response contains Cloudflare's challenge page
if (
diff --git a/apps/web/tests/misc/metatags-get-html.test.ts b/apps/web/tests/misc/metatags-get-html.test.ts
index 201c1855e1..4c873e64d7 100644
--- a/apps/web/tests/misc/metatags-get-html.test.ts
+++ b/apps/web/tests/misc/metatags-get-html.test.ts
@@ -39,7 +39,7 @@ const htmlStream = (totalBytes: number, highWaterMark = 1) => {
return { stream, stats };
};
-describe("getHtml response size cap", () => {
+describe("getHtml response size limit", () => {
beforeEach(() => {
vi.mocked(safeFetch).mockReset();
vi.spyOn(console, "warn").mockImplementation(() => {});
@@ -60,36 +60,26 @@ describe("getHtml response size cap", () => {
expect(html).toHaveLength(MAX_HTML_BYTES);
});
- it("returns null before reading the body when Content-Length is too large", async () => {
- const { stream, stats } = htmlStream(MAX_HTML_BYTES * 2, 0);
- vi.mocked(safeFetch).mockResolvedValue(
- new Response(stream, {
- headers: { "content-length": String(MAX_HTML_BYTES + 1) },
- }),
- );
-
- await expect(getHtml(URL)).resolves.toBeNull();
- expect(stats.cancelled).toBe(true);
- expect(stats.bytesPulled).toBe(0);
- expect(console.warn).toHaveBeenCalledWith(
- expect.stringContaining("Content-Length"),
- );
- });
-
- it("stops reading and returns null when a body without Content-Length passes the cap", async () => {
+ it("returns only the first MAX_HTML_BYTES of a larger body and stops reading", async () => {
const { stream, stats } = htmlStream(MAX_HTML_BYTES * 4);
vi.mocked(safeFetch).mockResolvedValue(new Response(stream));
- await expect(getHtml(URL)).resolves.toBeNull();
+ const html = await getHtml(URL);
+ expect(html).toHaveLength(MAX_HTML_BYTES);
expect(stats.cancelled).toBe(true);
- // The body is 4x the cap. Reading stops one chunk past the cap, and the
- // stream can pull one more chunk ahead before it is cancelled.
- expect(stats.bytesPulled).toBeLessThanOrEqual(
- MAX_HTML_BYTES + 2 * CHUNK_BYTES,
- );
- expect(console.warn).toHaveBeenCalledWith(
- expect.stringContaining(`passed ${MAX_HTML_BYTES} bytes`),
+ // The stream can pull one chunk ahead before it is cancelled.
+ expect(stats.bytesPulled).toBeLessThanOrEqual(MAX_HTML_BYTES + CHUNK_BYTES);
+ });
+
+ it("does not trust a Content-Length header that is too large", async () => {
+ const html = "<html><head><title>Dub</title></head></html>";
+ vi.mocked(safeFetch).mockResolvedValue(
+ new Response(html, {
+ headers: { "content-length": String(MAX_HTML_BYTES * 10) },
+ }),
);
+
+ await expect(getHtml(URL)).resolves.toBe(html);
});
it("decodes multi-byte characters split across chunks", async () => {