The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Stop reading the page at the end of the head in getHtml
Rafi4a2e4090fdad
diff --git a/apps/web/app/api/links/metatags/utils.ts b/apps/web/app/api/links/metatags/utils.tsindex 860e94651c..65ad4c8d14 100644--- a/apps/web/app/api/links/metatags/utils.ts+++ b/apps/web/app/api/links/metatags/utils.ts@@ -13,9 +13,12 @@ import { parse } from "node-html-parser";// head we checked was about 760 KB). Only this much of a page is read.export const MAX_HTML_BYTES = 2 * 1024 * 1024; // 2 MB-// Reads the body as UTF-8 text like response.text(), but reads at most maxBytes.-// A longer body is cut at maxBytes, because the meta tags are near the start.-const readTextWithLimit = async (response: Response, maxBytes: number) => {+const HEAD_END_REGEX = /<\/head\s*>/i;++// Reads the body as UTF-8 text up to the end of the <head>, because getMetaTags+// needs only the meta, title and link tags. Reads at most maxBytes, so a page+// with no </head> near the start is cut at maxBytes.+const readHtmlHead = async (response: Response, maxBytes: number) => {if (!response.body) {return "";}@@ -36,12 +39,21 @@ const readTextWithLimit = async (response: Response, maxBytes: number) => {value.byteLength > remaining ? value.subarray(0, remaining) : value;bytesRead += bytes.byteLength;- if (bytesRead >= maxBytes) {+ const atLimit = bytesRead >= maxBytes;+ // Search a few characters back, in case </head> is split across chunks.+ const searchFrom = Math.max(0, text.length - "</head>".length);+ text += decoder.decode(bytes, { stream: !atLimit });++ const headEnd = HEAD_END_REGEX.exec(text.slice(searchFrom));+ if (headEnd) {await reader.cancel().catch(() => {});- return text + decoder.decode(bytes);+ return text.slice(0, searchFrom + headEnd.index + headEnd[0].length);}- text += decoder.decode(bytes, { stream: true });+ if (atLimit) {+ await reader.cancel().catch(() => {});+ return text;+ }}return text + decoder.decode();@@ -62,8 +74,15 @@ export const getHtml = async (url: string) => {return null;}- // Content-Length can be missing or wrong, so the limit is on the stream.- const text = await readTextWithLimit(response, MAX_HTML_BYTES);+ // Cloudflare sets this header on its challenge pages. readHtmlHead reads only+ // the <head>, so the text check below can miss a challenge page.+ if (response.headers.get("cf-mitigated") === "challenge") {+ await response.body?.cancel().catch(() => {});+ console.warn(`Cloudflare challenge page detected for: ${url}`);+ return null;+ }++ const text = await readHtmlHead(response, MAX_HTML_BYTES);// Check if the response contains Cloudflare's challenge pageif (diff --git a/apps/web/tests/misc/metatags-get-html.test.ts b/apps/web/tests/misc/metatags-get-html.test.tsindex 4c873e64d7..aa84f80a92 100644--- a/apps/web/tests/misc/metatags-get-html.test.ts+++ b/apps/web/tests/misc/metatags-get-html.test.ts@@ -45,11 +45,13 @@ describe("getHtml response size limit", () => {vi.spyOn(console, "warn").mockImplementation(() => {});});- it("returns the HTML of a normal page", async () => {- const html = "<html><head><title>Dub</title></head></html>";+ it("returns the HTML of a normal page up to the end of the head", async () => {+ const html = "<html><head><title>Dub</title></head><body>Hi</body></html>";vi.mocked(safeFetch).mockResolvedValue(new Response(html));- await expect(getHtml(URL)).resolves.toBe(html);+ await expect(getHtml(URL)).resolves.toBe(+ "<html><head><title>Dub</title></head>",+ );});it("returns a body of exactly the cap", async () => {@@ -72,7 +74,7 @@ describe("getHtml response size limit", () => {});it("does not trust a Content-Length header that is too large", async () => {- const html = "<html><head><title>Dub</title></head></html>";+ const html = "<html><head><title>Dub</title></head>";vi.mocked(safeFetch).mockResolvedValue(new Response(html, {headers: { "content-length": String(MAX_HTML_BYTES * 10) },@@ -82,6 +84,71 @@ describe("getHtml response size limit", () => {await expect(getHtml(URL)).resolves.toBe(html);});+ it("stops reading at the end of the head", async () => {+ const head = new TextEncoder().encode(+ "<html><head><title>Dub</title></head><body>",+ );+ const { stream: body, stats } = htmlStream(MAX_HTML_BYTES * 4, 0);+ const bodyReader = body.getReader();+ let headSent = false;+ const stream = new ReadableStream<Uint8Array>(+ {+ async pull(controller) {+ if (!headSent) {+ headSent = true;+ controller.enqueue(head);+ return;+ }+ const { done, value } = await bodyReader.read();+ if (done) {+ controller.close();+ } else {+ controller.enqueue(value);+ }+ },+ cancel() {+ return bodyReader.cancel();+ },+ },+ { highWaterMark: 0 },+ );+ vi.mocked(safeFetch).mockResolvedValue(new Response(stream));++ await expect(getHtml(URL)).resolves.toBe(+ "<html><head><title>Dub</title></head>",+ );+ expect(stats.bytesPulled).toBe(0);+ expect(stats.cancelled).toBe(true);+ });++ it("finds an uppercase </HEAD> split across chunks", async () => {+ const encoder = new TextEncoder();+ const stream = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(+ encoder.encode("<HTML><HEAD><TITLE>Dub</TITLE></HE"),+ );+ controller.enqueue(encoder.encode("AD><BODY>Hi</BODY></HTML>"));+ controller.close();+ },+ });+ vi.mocked(safeFetch).mockResolvedValue(new Response(stream));++ await expect(getHtml(URL)).resolves.toBe(+ "<HTML><HEAD><TITLE>Dub</TITLE></HEAD>",+ );+ });++ it("returns null for a Cloudflare challenge page", async () => {+ vi.mocked(safeFetch).mockResolvedValue(+ new Response("<html><head><title>Just a moment...</title></head>", {+ headers: { "cf-mitigated": "challenge" },+ }),+ );++ await expect(getHtml(URL)).resolves.toBeNull();+ });+it("decodes multi-byte characters split across chunks", async () => {const bytes = new TextEncoder().encode("<title>café 🔗</title>");const stream = new ReadableStream<Uint8Array>({