Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Cap the HTML response size in the metatags getHtml

Rafi461b4f21cdc0
diff --git a/apps/web/app/api/links/metatags/utils.ts b/apps/web/app/api/links/metatags/utils.ts
index 4350cf0058..01d2de2e91 100644
--- a/apps/web/app/api/links/metatags/utils.ts
+++ b/apps/web/app/api/links/metatags/utils.ts
@@ -9,6 +9,40 @@ import { waitUntil } from "@vercel/functions";
import he from "he";
import { parse } from "node-html-parser";
+// Meta tags are in the <head>, which ends before 1 MB on real pages. Some full
+// pages (long Wikipedia articles) are near 3 MB, so the cap leaves room for them.
+export const MAX_HTML_BYTES = 5 * 1024 * 1024; // 5 MB
+
+// Reads the body as UTF-8 text like response.text(), but stops and returns
+// null when the body passes maxBytes.
+const readTextWithLimit = async (response: Response, maxBytes: number) => {
+ if (!response.body) {
+ return "";
+ }
+
+ const reader = response.body.getReader();
+ const decoder = new TextDecoder();
+ let bytesRead = 0;
+ let text = "";
+
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) {
+ break;
+ }
+
+ bytesRead += value.byteLength;
+ if (bytesRead > maxBytes) {
+ await reader.cancel().catch(() => {});
+ return null;
+ }
+
+ text += decoder.decode(value, { stream: true });
+ }
+
+ return text + decoder.decode();
+};
+
export const getHtml = async (url: string) => {
try {
const response = await safeFetch(url);
@@ -24,7 +58,23 @@ export const getHtml = async (url: string) => {
return null;
}
- const text = await response.text();
+ const contentLength = Number(response.headers.get("content-length"));
+ if (contentLength > MAX_HTML_BYTES) {
+ await response.body?.cancel().catch(() => {});
+ console.warn(
+ `HTML response too large (Content-Length: ${contentLength}) for URL: ${url}`,
+ );
+ return null;
+ }
+
+ // Content-Length can be missing or wrong, so the stream is capped as well.
+ const text = await readTextWithLimit(response, MAX_HTML_BYTES);
+ if (text === null) {
+ console.warn(
+ `HTML response passed ${MAX_HTML_BYTES} bytes for URL: ${url}`,
+ );
+ return null;
+ }
// Check if the response contains Cloudflare's challenge page
if (