The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Keep the first 2 MB of a large page in getHtml
Rafi09b262a0768e
diff --git a/apps/web/app/api/links/metatags/utils.ts b/apps/web/app/api/links/metatags/utils.tsindex 01d2de2e91..860e94651c 100644--- a/apps/web/app/api/links/metatags/utils.ts+++ b/apps/web/app/api/links/metatags/utils.ts@@ -9,12 +9,12 @@ import { waitUntil } from "@vercel/functions";import he from "he";import { parse } from "node-html-parser";-// Meta tags are in the <head>, which ends before 1 MB on real pages. Some full-// pages (long Wikipedia articles) are near 3 MB, so the cap leaves room for them.-export const MAX_HTML_BYTES = 5 * 1024 * 1024; // 5 MB+// Meta tags are in the <head>, which ends before 1 MB on real pages (the largest+// head we checked was about 760 KB). Only this much of a page is read.+export const MAX_HTML_BYTES = 2 * 1024 * 1024; // 2 MB-// Reads the body as UTF-8 text like response.text(), but stops and returns-// null when the body passes maxBytes.+// Reads the body as UTF-8 text like response.text(), but reads at most maxBytes.+// A longer body is cut at maxBytes, because the meta tags are near the start.const readTextWithLimit = async (response: Response, maxBytes: number) => {if (!response.body) {return "";@@ -31,13 +31,17 @@ const readTextWithLimit = async (response: Response, maxBytes: number) => {break;}- bytesRead += value.byteLength;- if (bytesRead > maxBytes) {+ const remaining = maxBytes - bytesRead;+ const bytes =+ value.byteLength > remaining ? value.subarray(0, remaining) : value;+ bytesRead += bytes.byteLength;++ if (bytesRead >= maxBytes) {await reader.cancel().catch(() => {});- return null;+ return text + decoder.decode(bytes);}- text += decoder.decode(value, { stream: true });+ text += decoder.decode(bytes, { stream: true });}return text + decoder.decode();@@ -58,23 +62,8 @@ export const getHtml = async (url: string) => {return null;}- const contentLength = Number(response.headers.get("content-length"));- if (contentLength > MAX_HTML_BYTES) {- await response.body?.cancel().catch(() => {});- console.warn(- `HTML response too large (Content-Length: ${contentLength}) for URL: ${url}`,- );- return null;- }-- // Content-Length can be missing or wrong, so the stream is capped as well.+ // Content-Length can be missing or wrong, so the limit is on the stream.const text = await readTextWithLimit(response, MAX_HTML_BYTES);- if (text === null) {- console.warn(- `HTML response passed ${MAX_HTML_BYTES} bytes for URL: ${url}`,- );- return null;- }// Check if the response contains Cloudflare's challenge pageif (diff --git a/apps/web/tests/misc/metatags-get-html.test.ts b/apps/web/tests/misc/metatags-get-html.test.tsindex 201c1855e1..4c873e64d7 100644--- a/apps/web/tests/misc/metatags-get-html.test.ts+++ b/apps/web/tests/misc/metatags-get-html.test.ts@@ -39,7 +39,7 @@ const htmlStream = (totalBytes: number, highWaterMark = 1) => {return { stream, stats };};-describe("getHtml response size cap", () => {+describe("getHtml response size limit", () => {beforeEach(() => {vi.mocked(safeFetch).mockReset();vi.spyOn(console, "warn").mockImplementation(() => {});@@ -60,36 +60,26 @@ describe("getHtml response size cap", () => {expect(html).toHaveLength(MAX_HTML_BYTES);});- it("returns null before reading the body when Content-Length is too large", async () => {- const { stream, stats } = htmlStream(MAX_HTML_BYTES * 2, 0);- vi.mocked(safeFetch).mockResolvedValue(- new Response(stream, {- headers: { "content-length": String(MAX_HTML_BYTES + 1) },- }),- );-- await expect(getHtml(URL)).resolves.toBeNull();- expect(stats.cancelled).toBe(true);- expect(stats.bytesPulled).toBe(0);- expect(console.warn).toHaveBeenCalledWith(- expect.stringContaining("Content-Length"),- );- });-- it("stops reading and returns null when a body without Content-Length passes the cap", async () => {+ it("returns only the first MAX_HTML_BYTES of a larger body and stops reading", async () => {const { stream, stats } = htmlStream(MAX_HTML_BYTES * 4);vi.mocked(safeFetch).mockResolvedValue(new Response(stream));- await expect(getHtml(URL)).resolves.toBeNull();+ const html = await getHtml(URL);+ expect(html).toHaveLength(MAX_HTML_BYTES);expect(stats.cancelled).toBe(true);- // The body is 4x the cap. Reading stops one chunk past the cap, and the- // stream can pull one more chunk ahead before it is cancelled.- expect(stats.bytesPulled).toBeLessThanOrEqual(- MAX_HTML_BYTES + 2 * CHUNK_BYTES,- );- expect(console.warn).toHaveBeenCalledWith(- expect.stringContaining(`passed ${MAX_HTML_BYTES} bytes`),+ // The stream can pull one chunk ahead before it is cancelled.+ expect(stats.bytesPulled).toBeLessThanOrEqual(MAX_HTML_BYTES + CHUNK_BYTES);+ });++ it("does not trust a Content-Length header that is too large", async () => {+ const html = "<html><head><title>Dub</title></head></html>";+ vi.mocked(safeFetch).mockResolvedValue(+ new Response(html, {+ headers: { "content-length": String(MAX_HTML_BYTES * 10) },+ }),);++ await expect(getHtml(URL)).resolves.toBe(html);});it("decodes multi-byte characters split across chunks", async () => {