Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Stop reading the page at the end of the head in getHtml

Rafi4a2e4090fdad
diff --git a/apps/web/app/api/links/metatags/utils.ts b/apps/web/app/api/links/metatags/utils.ts
index 860e94651c..65ad4c8d14 100644
--- a/apps/web/app/api/links/metatags/utils.ts
+++ b/apps/web/app/api/links/metatags/utils.ts
@@ -13,9 +13,12 @@ import { parse } from "node-html-parser";
// head we checked was about 760 KB). Only this much of a page is read.
export const MAX_HTML_BYTES = 2 * 1024 * 1024; // 2 MB
-// Reads the body as UTF-8 text like response.text(), but reads at most maxBytes.
-// A longer body is cut at maxBytes, because the meta tags are near the start.
-const readTextWithLimit = async (response: Response, maxBytes: number) => {
+const HEAD_END_REGEX = /<\/head\s*>/i;
+
+// Reads the body as UTF-8 text up to the end of the <head>, because getMetaTags
+// needs only the meta, title and link tags. Reads at most maxBytes, so a page
+// with no </head> near the start is cut at maxBytes.
+const readHtmlHead = async (response: Response, maxBytes: number) => {
if (!response.body) {
return "";
}
@@ -36,12 +39,21 @@ const readTextWithLimit = async (response: Response, maxBytes: number) => {
value.byteLength > remaining ? value.subarray(0, remaining) : value;
bytesRead += bytes.byteLength;
- if (bytesRead >= maxBytes) {
+ const atLimit = bytesRead >= maxBytes;
+ // Search a few characters back, in case </head> is split across chunks.
+ const searchFrom = Math.max(0, text.length - "</head>".length);
+ text += decoder.decode(bytes, { stream: !atLimit });
+
+ const headEnd = HEAD_END_REGEX.exec(text.slice(searchFrom));
+ if (headEnd) {
await reader.cancel().catch(() => {});
- return text + decoder.decode(bytes);
+ return text.slice(0, searchFrom + headEnd.index + headEnd[0].length);
}
- text += decoder.decode(bytes, { stream: true });
+ if (atLimit) {
+ await reader.cancel().catch(() => {});
+ return text;
+ }
}
return text + decoder.decode();
@@ -62,8 +74,15 @@ export const getHtml = async (url: string) => {
return null;
}
- // Content-Length can be missing or wrong, so the limit is on the stream.
- const text = await readTextWithLimit(response, MAX_HTML_BYTES);
+ // Cloudflare sets this header on its challenge pages. readHtmlHead reads only
+ // the <head>, so the text check below can miss a challenge page.
+ if (response.headers.get("cf-mitigated") === "challenge") {
+ await response.body?.cancel().catch(() => {});
+ console.warn(`Cloudflare challenge page detected for: ${url}`);
+ return null;
+ }
+
+ const text = await readHtmlHead(response, MAX_HTML_BYTES);
// Check if the response contains Cloudflare's challenge page
if (
diff --git a/apps/web/tests/misc/metatags-get-html.test.ts b/apps/web/tests/misc/metatags-get-html.test.ts
index 4c873e64d7..aa84f80a92 100644
--- a/apps/web/tests/misc/metatags-get-html.test.ts
+++ b/apps/web/tests/misc/metatags-get-html.test.ts
@@ -45,11 +45,13 @@ describe("getHtml response size limit", () => {
vi.spyOn(console, "warn").mockImplementation(() => {});
});
- it("returns the HTML of a normal page", async () => {
- const html = "<html><head><title>Dub</title></head></html>";
+ it("returns the HTML of a normal page up to the end of the head", async () => {
+ const html = "<html><head><title>Dub</title></head><body>Hi</body></html>";
vi.mocked(safeFetch).mockResolvedValue(new Response(html));
- await expect(getHtml(URL)).resolves.toBe(html);
+ await expect(getHtml(URL)).resolves.toBe(
+ "<html><head><title>Dub</title></head>",
+ );
});
it("returns a body of exactly the cap", async () => {
@@ -72,7 +74,7 @@ describe("getHtml response size limit", () => {
});
it("does not trust a Content-Length header that is too large", async () => {
- const html = "<html><head><title>Dub</title></head></html>";
+ const html = "<html><head><title>Dub</title></head>";
vi.mocked(safeFetch).mockResolvedValue(
new Response(html, {
headers: { "content-length": String(MAX_HTML_BYTES * 10) },
@@ -82,6 +84,71 @@ describe("getHtml response size limit", () => {
await expect(getHtml(URL)).resolves.toBe(html);
});
+ it("stops reading at the end of the head", async () => {
+ const head = new TextEncoder().encode(
+ "<html><head><title>Dub</title></head><body>",
+ );
+ const { stream: body, stats } = htmlStream(MAX_HTML_BYTES * 4, 0);
+ const bodyReader = body.getReader();
+ let headSent = false;
+ const stream = new ReadableStream<Uint8Array>(
+ {
+ async pull(controller) {
+ if (!headSent) {
+ headSent = true;
+ controller.enqueue(head);
+ return;
+ }
+ const { done, value } = await bodyReader.read();
+ if (done) {
+ controller.close();
+ } else {
+ controller.enqueue(value);
+ }
+ },
+ cancel() {
+ return bodyReader.cancel();
+ },
+ },
+ { highWaterMark: 0 },
+ );
+ vi.mocked(safeFetch).mockResolvedValue(new Response(stream));
+
+ await expect(getHtml(URL)).resolves.toBe(
+ "<html><head><title>Dub</title></head>",
+ );
+ expect(stats.bytesPulled).toBe(0);
+ expect(stats.cancelled).toBe(true);
+ });
+
+ it("finds an uppercase </HEAD> split across chunks", async () => {
+ const encoder = new TextEncoder();
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(
+ encoder.encode("<HTML><HEAD><TITLE>Dub</TITLE></HE"),
+ );
+ controller.enqueue(encoder.encode("AD><BODY>Hi</BODY></HTML>"));
+ controller.close();
+ },
+ });
+ vi.mocked(safeFetch).mockResolvedValue(new Response(stream));
+
+ await expect(getHtml(URL)).resolves.toBe(
+ "<HTML><HEAD><TITLE>Dub</TITLE></HEAD>",
+ );
+ });
+
+ it("returns null for a Cloudflare challenge page", async () => {
+ vi.mocked(safeFetch).mockResolvedValue(
+ new Response("<html><head><title>Just a moment...</title></head>", {
+ headers: { "cf-mitigated": "challenge" },
+ }),
+ );
+
+ await expect(getHtml(URL)).resolves.toBeNull();
+ });
+
it("decodes multi-byte characters split across chunks", async () => {
const bytes = new TextEncoder().encode("<title>café 🔗</title>");
const stream = new ReadableStream<Uint8Array>({