Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Stream link-preview fetch to head-end for large pages (PRD-7495) (#2776)
Andrew Mikofalvyb6e273042dfa
diff --git a/.changeset/prd-7495-headfirst-link-preview.md b/.changeset/prd-7495-headfirst-link-preview.mdnew file mode 100644index 000000000..86c8a0423--- /dev/null+++ b/.changeset/prd-7495-headfirst-link-preview.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Link previews now load for content-heavy pages like GitHub and Wikipedia: the fetcher streams to the end of <head> instead of rejecting once the page body exceeds the 512 KB cap.diff --git a/packages/server/src/link-preview/guarded-fetch.test.ts b/packages/server/src/link-preview/guarded-fetch.test.tsindex aac37542a..4af60d8dc 100644--- a/packages/server/src/link-preview/guarded-fetch.test.ts+++ b/packages/server/src/link-preview/guarded-fetch.test.ts@@ -1,8 +1,9 @@-import { afterAll, beforeAll, describe, expect, test } from 'bun:test';import { createServer, type Server } from 'node:http';import { gzipSync } from 'node:zlib';+import { afterAll, beforeAll, describe, expect, test } from 'vitest';import { listenOnLoopback } from '../loopback-rig-test-helpers.ts';import {+ createHeadEndScanner,DEFAULT_MAX_BYTES,DEFAULT_MAX_REDIRECTS,DEFAULT_TIMEOUT_MS,@@ -99,6 +100,78 @@ describe('guardedFetch admission (real classifier, no network reached)', () => {});});+describe('createHeadEndScanner (chunk-boundary-safe head-end detection)', () => {+ const bytes = (text: string) => new Uint8Array(Buffer.from(text, 'latin1'));++ test('finds </head> within a single chunk and reports the offset past it', () => {+ const scan = createHeadEndScanner();+ const html = '<html><head><title>t</title></head><body>x</body>';+ expect(scan(bytes(html))).toBe(html.indexOf('</head>') + '</head>'.length);+ });++ test.each([+ 1, 2, 3, 4, 5, 6,+ ])('finds </head> split across a chunk boundary after %i marker byte(s)', (splitAt) => {+ const scan = createHeadEndScanner();+ const marker = '</head>';+ expect(scan(bytes(`<html><head><title>t</title>${marker.slice(0, splitAt)}`))).toBe(-1);+ const rest = `${marker.slice(splitAt)}<body>tail`;+ expect(scan(bytes(rest))).toBe(marker.length - splitAt);+ });++ test('finds an opening <body> when no </head> precedes it', () => {+ const scan = createHeadEndScanner();+ const html = '<html><head><title>t</title><body>x';+ expect(scan(bytes(html))).toBe(html.indexOf('<body') + '<body'.length + 1);+ });++ test('finds an opening <body with attributes (whitespace terminator)', () => {+ const scan = createHeadEndScanner();+ const html = '<html><head><body class="a">x';+ expect(scan(bytes(html))).toBe(html.indexOf('<body') + '<body '.length);+ });++ test('finds <body split across a chunk boundary', () => {+ const scan = createHeadEndScanner();+ expect(scan(bytes('<html><head><title>t</title><bo'))).toBe(-1);+ expect(scan(bytes('dy>tail'))).toBe(3);+ });++ test('holds a chunk-final bare <body until the next chunk settles its terminator', () => {+ const scan = createHeadEndScanner();+ expect(scan(bytes('<html><head><body'))).toBe(-1);+ expect(scan(bytes('>x'))).toBe(1);+ });++ test('is ASCII case-insensitive (</HEAD>, <BODY>)', () => {+ const upperHead = createHeadEndScanner();+ expect(upperHead(bytes('<HTML><HEAD><TITLE>T</TITLE></HEAD>'))).toBe(+ '<HTML><HEAD><TITLE>T</TITLE></HEAD>'.length,+ );+ const upperBody = createHeadEndScanner();+ expect(upperBody(bytes('<HTML><BODY>x'))).toBe('<HTML><BODY'.length + 1);+ });++ test('reports the earlier marker when a <body opens before a later </head>', () => {+ const scan = createHeadEndScanner();+ const html = '<x><body>then</head>';+ expect(scan(bytes(html))).toBe(html.indexOf('<body') + '<body'.length + 1);+ });++ test('does not match a custom element like <bodyguard>, even across chunks', () => {+ const scan = createHeadEndScanner();+ expect(scan(bytes('<html><bodyg'))).toBe(-1);+ expect(scan(bytes('uard>content'))).toBe(-1);+ });++ test('returns -1 across many marker-free chunks', () => {+ const scan = createHeadEndScanner();+ for (let i = 0; i < 20; i++) {+ expect(scan(bytes('<p>plain markup with heads and bodies spelled out</p>'))).toBe(-1);+ }+ });+});+describe('guardedFetch against a loopback rig (real socket)', () => {let server: Server;let port: number;@@ -111,6 +184,16 @@ describe('guardedFetch against a loopback rig (real socket)', () => {referer?: string;} | null = null;+ const HUGE_PAGE_HEAD =+ '<html><head><title>Huge Page</title><meta name="description" content="streams forever"></head>';+ let hugeStream = { closedEarly: false, finishedAll: false };+ // Resolves when the huge-streaming socket closes, so the test waits on that+ // event instead of polling a flag. Reassigned per request, with the resolver+ // captured in that request's own close handler, so a late close from an+ // earlier run resolves its own promise rather than the current run's (keeps+ // the rig correct if this suite ever gains test.retry or test.concurrent).+ let hugeClosed: Promise<void> = Promise.resolve();+// The rig binds loopback; a public hostname resolves to it via the injected// resolver, and the guard is told to treat that one loopback address as// public so post-admission behavior can be exercised over a real socket.@@ -193,6 +276,96 @@ describe('guardedFetch against a loopback rig (real socket)', () => {res.end('{"not":"html"}');return;}+ if (path === '/huge-streaming') {+ // GitHub-shaped page: the head arrives in the first write, then a body+ // far past the byte cap keeps streaming until the client hangs up.+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.write(HUGE_PAGE_HEAD);+ const observed = { closedEarly: false, finishedAll: false };+ hugeStream = observed;+ const closed = Promise.withResolvers<void>();+ hugeClosed = closed.promise;+ const filler = Buffer.alloc(64 * 1024, 0x78);+ let written = 0;+ const timer = setInterval(() => {+ if (res.destroyed || res.writableEnded) return;+ if (written >= 8 * 1024 * 1024) {+ observed.finishedAll = true;+ clearInterval(timer);+ res.end();+ return;+ }+ written += filler.byteLength;+ res.write(filler);+ }, 1);+ res.on('close', () => {+ clearInterval(timer);+ if (!res.writableFinished) observed.closedEarly = true;+ closed.resolve();+ });+ return;+ }+ if (path === '/marker-in-head-script') {+ // A </head> spelled inside a head-level <script> string, then the REAL+ // head. A context-blind byte scan stops at the script-internal marker+ // and loses the title; the extractor-shared boundary skips the script.+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.end(+ `<html><head><script>var s='</head><body>';</script>` ++ `<title>RealTitle</title></head><body>${'z'.repeat(1024)}</body></html>`,+ );+ return;+ }+ if (path === '/head-end-past-cap-multichunk') {+ // A non-marker preamble in one write, then the head-end in a later+ // write. Only counting the preamble (`received`) pushes the total past+ // the cap, so the guard must add it to the in-chunk offset.+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.write('a'.repeat(80));+ setTimeout(() => res.end(`${'b'.repeat(37)}</head>tail`), 20);+ return;+ }+ if (path === '/head-split-across-writes') {+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.write('<html><head><title>Split</title></he');+ setTimeout(() => res.end('ad><body>after the boundary</body>'), 20);+ return;+ }+ if (path === '/body-open-no-head-close') {+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.end(`<html><head><title>NoClose</title><body class="x">${'y'.repeat(64 * 1024)}`);+ return;+ }+ if (path === '/upper-head') {+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.end(`<HTML><HEAD><TITLE>Caps</TITLE></HEAD><BODY>${'z'.repeat(64 * 1024)}`);+ return;+ }+ if (path === '/marker-after-cap') {+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.end(`${'x'.repeat(2048)}</head>`);+ return;+ }+ if (path === '/small-no-markers') {+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.end('<html><title>Whole</title><p>every byte returned</p>');+ return;+ }+ if (path === '/json-with-marker-bytes') {+ res.writeHead(200, { 'Content-Type': 'application/json' });+ res.end('{"snippet":"</head><body>","trailer":"kept"}');+ return;+ }+ if (path === '/gzip-huge-head-first') {+ // Tiny on the wire; decompressed far past the cap used by the test —+ // but the head ends early, so the scan (which runs on DECOMPRESSED+ // bytes) must admit it where the old whole-body read rejected it.+ res.writeHead(200, { 'Content-Type': 'text/html', 'Content-Encoding': 'gzip' });+ res.end(+ gzipSync(`<html><head><title>Zipped</title></head><body>${'w'.repeat(200 * 1024)}`),+ );+ return;+ }if (path === '/slow') {const timer = setTimeout(() => {if (!res.writableEnded) {@@ -335,4 +508,124 @@ describe('guardedFetch against a loopback rig (real socket)', () => {});expect(result).toEqual({ ok: false, reason: 'timeout' });});++ test('succeeds on a huge streaming page whose head arrives first, cancelling the rest', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/huge-streaming`, withRig);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ // The body is the exact head prefix — nothing past head-end is buffered,+ // regardless of how the runtime coalesced the filler into chunks.+ expect(new TextDecoder().decode(result.body)).toBe(HUGE_PAGE_HEAD);+ expect(result.body.byteLength).toBeLessThanOrEqual(DEFAULT_MAX_BYTES);+ // The reader hung up on the still-streaming remainder rather than consuming+ // the multi-MB body: wait for the socket-close event, failing loudly if it+ // never arrives instead of polling a flag on a fixed interval.+ let timer: ReturnType<typeof setTimeout> | undefined;+ const timeout = new Promise<'timeout'>((resolve) => {+ timer = setTimeout(() => resolve('timeout'), 2000);+ });+ const outcome = await Promise.race([hugeClosed.then(() => 'closed' as const), timeout]);+ clearTimeout(timer);+ expect(outcome).toBe('closed');+ expect(hugeStream.closedEarly).toBe(true);+ expect(hugeStream.finishedAll).toBe(false);+ });++ test('keeps the real head when a </head> is spelled inside a head-level <script>', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/marker-in-head-script`, withRig);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ // The returned prefix is EXACTLY the real head (reaching past the+ // script-internal `</head>` to the real one), so the metadata extractor+ // still sees the title. An exact match kills both mutants: truncated-too-early+ // (stopping at the script-internal marker drops the title) and+ // truncated-too-late (the whole 1 KB-body response also contains both the+ // title and the script string, so a substring check would pass on it too).+ const body = new TextDecoder().decode(result.body);+ expect(body).toBe(+ "<html><head><script>var s='</head><body>';</script><title>RealTitle</title></head>",+ );+ });++ test('detects a </head> split across two response writes', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/head-split-across-writes`, {+ ...withRig,+ maxBytes: 4 * 1024,+ });+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(new TextDecoder().decode(result.body)).toBe('<html><head><title>Split</title></head>');+ });++ test('terminates at an opening <body when the page never closes its head', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/body-open-no-head-close`, {+ ...withRig,+ maxBytes: 4 * 1024,+ });+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ // 64KB of body filler against a 4KB cap: only head-end termination admits it.+ expect(new TextDecoder().decode(result.body)).toBe('<html><head><title>NoClose</title><body ');+ });++ test('matches head-end case-insensitively (</HEAD>)', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/upper-head`, {+ ...withRig,+ maxBytes: 4 * 1024,+ });+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(new TextDecoder().decode(result.body)).toBe('<HTML><HEAD><TITLE>Caps</TITLE></HEAD>');+ });++ test('still rejects as oversized when the cap is reached before head-end', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/marker-after-cap`, {+ ...withRig,+ maxBytes: 1024,+ });+ expect(result).toEqual({ ok: false, reason: 'oversized' });+ });++ test('rejects as oversized when head-end clears the cap only after earlier chunks count', async () => {+ // 80-byte preamble in chunk one, then </head> at offset 44 of chunk two:+ // oversized only if the guard adds the 80 already received to the in-chunk+ // offset (80 + 44 = 124 > 100). A check that dropped `received` would admit+ // an over-cap body across writes.+ const result = await guardedFetch(`http://rig.example:${port}/head-end-past-cap-multichunk`, {+ ...withRig,+ maxBytes: 100,+ });+ expect(result).toEqual({ ok: false, reason: 'oversized' });+ });++ test('returns an under-cap page with no head markers whole, as before', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/small-no-markers`, withRig);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(new TextDecoder().decode(result.body)).toBe(+ '<html><title>Whole</title><p>every byte returned</p>',+ );+ });++ test('reads non-HTML content in full even when it contains marker-shaped bytes', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/json-with-marker-bytes`, {+ ...withRig,+ allowContentType: (mimeType) => mimeType === 'application/json',+ });+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(new TextDecoder().decode(result.body)).toBe(+ '{"snippet":"</head><body>","trailer":"kept"}',+ );+ });++ test('applies the head-end scan to DECOMPRESSED bytes (gzip page far past the cap)', async () => {+ const result = await guardedFetch(`http://rig.example:${port}/gzip-huge-head-first`, {+ ...withRig,+ maxBytes: 8 * 1024,+ });+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(new TextDecoder().decode(result.body)).toBe('<html><head><title>Zipped</title></head>');+ });});diff --git a/packages/server/src/link-preview/guarded-fetch.ts b/packages/server/src/link-preview/guarded-fetch.tsindex 313064e9d..4fc569284 100644--- a/packages/server/src/link-preview/guarded-fetch.ts+++ b/packages/server/src/link-preview/guarded-fetch.ts@@ -30,7 +30,14 @@* re-run the full scheme+resolve+validate admission on each hop's target,* bounded to a small hop count. Response reads are bounded on both time* (AbortSignal) and DECOMPRESSED size, and admit only allowlisted content- * types. Decompression contract: the request advertises+ * types. HTML reads additionally stop at head-end (`</head>` close or `<body`+ * open, scanned incrementally as decompressed bytes arrive): everything the+ * metadata extractor reads lives in the head, so the buffered prefix is+ * returned as the successful body and the rest of the page is never consumed.+ * The byte cap therefore rejects a page only when it is reached BEFORE+ * head-end — content-heavy pages whose head arrives early preview fine, while+ * bounded work is preserved (never more than the cap buffered, no reads past+ * head-end). Decompression contract: the request advertises* `Accept-Encoding: identity`; if a server compresses anyway, the body is* streamed through node:zlib (gzip/deflate/br) and the byte cap is applied to* the DECOMPRESSED output — the decompression-bomb guard — while unknown@@ -51,13 +58,17 @@ import type { LookupFunction } from 'node:net';import type { Transform } from 'node:stream';import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';import { getLogger } from '../logger.ts';+import { findHeadEndOffset } from './html-metadata.ts';import { classifyHost, isPublicUnicastIp } from './ip-classifier.ts';const logger = getLogger('link-preview.guarded-fetch');const USER_AGENT = 'OpenKnowledge-LinkPreview/1.x';-/** Decompressed-body ceiling; reads abort once a response exceeds it. */+/**+ * Decompressed-body ceiling; a read aborts as oversized when it is reached+ * before head-end (for HTML) or before end-of-body (anything else).+ */export const DEFAULT_MAX_BYTES = 512 * 1024;/** Total wall-clock budget across all redirect hops and the body read. */export const DEFAULT_TIMEOUT_MS = 5000;@@ -81,7 +92,12 @@ export type GuardRejectReason =/** @lintignore Union member of the exported GuardedFetchResult; no direct importer. */export interface GuardedFetchSuccess {ok: true;- /** Decompressed response bytes, guaranteed within the byte cap. */+ /**+ * Decompressed response bytes, guaranteed within the byte cap. For an HTML+ * response this is the prefix up to head-end when the page declares one —+ * sufficient for the head-only metadata extractor, and the reason a page+ * whose full body exceeds the cap can still succeed.+ */body: Uint8Array;/** Lowercased mime type with parameters stripped (e.g. `text/html`). */contentType: string;@@ -341,10 +357,42 @@ function makeDecompressor(encoding: string): Transform | 'identity' | null {return null;}+/**+ * Incremental head-end detector for a streaming HTML body. Feed decompressed+ * chunks in order; each call returns the offset into the CURRENT chunk just+ * past the head-end boundary (`</head>`'s closing `>`, or the terminator that+ * ends a `<body` open tag's name, so an attributed `<body class="a">` reports+ * the offset just past that terminator, mid-tag), or -1 while the head has not+ * ended. Chunks are accumulated and re-scanned by `findHeadEndOffset` (the same+ * context-aware walk the metadata extractor uses), so markup-shaped bytes inside+ * a head-level script, comment, quoted attribute value, or `<title>` are skipped+ * rather than mistaken for the end, and a boundary split across chunks is+ * detected once its bytes arrive. The accumulated view is latin1 (1:1 with+ * bytes; the markup tokens are ASCII), so the offset it yields is a true byte+ * offset.+ *+ * Re-scanning the whole accumulation per chunk is O(n^2) in the accumulated+ * size; the fetch layer's byte cap (`DEFAULT_MAX_BYTES`) bounds that+ * accumulation, so raising the cap raises this scan's worst-case cost+ * quadratically.+ */+export function createHeadEndScanner(): (chunk: Uint8Array) => number {+ let html = '';+ return (chunk) => {+ const consumed = html.length;+ html += Buffer.from(chunk).toString('latin1');+ const end = findHeadEndOffset(html);+ // The previous accumulation held no boundary, so a newly found one always+ // lands within this chunk: its offset is in (consumed, html.length].+ return end === -1 ? -1 : end - consumed;+ };+}+function readCappedBody(message: IncomingMessage,maxBytes: number,signal: AbortSignal,+ scanHeadEnd: boolean,): Promise<{ ok: true; body: Uint8Array } | { ok: false; reason: GuardRejectReason }> {const encoding = (message.headers['content-encoding'] ?? '').trim().toLowerCase();const decompressor = makeDecompressor(encoding);@@ -358,6 +406,7 @@ function readCappedBody(return new Promise((settle) => {const chunks: Buffer[] = [];+ const findHeadEnd = scanHeadEnd ? createHeadEndScanner() : null;let received = 0;let settled = false;@@ -384,6 +433,24 @@ function readCappedBody();}source.on('data', (chunk: Buffer) => {+ if (findHeadEnd) {+ const endInChunk = findHeadEnd(chunk);+ if (endInChunk !== -1) {+ // Head-end seen: the head prefix is a complete successful body.+ // Realize the head-first contract (only a cap reached BEFORE head-end+ // is oversized) by cap-checking just the bytes up to the marker+ // (`received + endInChunk`), then truncating there and stopping all+ // further reads via teardown. The whole-chunk cap check below is+ // bypassed, so bytes past head-end never count toward the cap.+ if (received + endInChunk > maxBytes) {+ resolveOnce({ ok: false, reason: 'oversized' });+ return;+ }+ chunks.push(chunk.subarray(0, endInChunk));+ resolveOnce({ ok: true, body: new Uint8Array(Buffer.concat(chunks)) });+ return;+ }+ }received += chunk.byteLength;if (received > maxBytes) {resolveOnce({ ok: false, reason: 'oversized' });@@ -404,9 +471,12 @@ function rejectWith(reason: GuardRejectReason): GuardedFetchFailure {}/**- * Fetch a URL under the SSRF guard. Returns the bounded response body plus the- * derived content-type and final URL on success, or a bounded reason code on- * any violation — never a partial body.+ * Fetch a URL under the SSRF guard. On success returns the bounded response+ * body plus the derived content-type and final URL; on any violation, a bounded+ * reason code. For an HTML response the body is the head prefix (everything up+ * to `</head>`/`<body>`) — all the metadata extractor reads — while any other+ * admitted content (the favicon path's images) is returned whole. A failure+ * never yields a partial body.*/export async function guardedFetch(rawUrl: string,@@ -453,7 +523,15 @@ export async function guardedFetch(return rejectWith('non-html');}- const bodyResult = await readCappedBody(response, maxBytes, signal);+ // Head-first streaming applies only to HTML pages; any other admitted+ // content (the favicon path's images) may legitimately contain+ // marker-shaped bytes and must be read in full.+ const bodyResult = await readCappedBody(+ response,+ maxBytes,+ signal,+ contentType === 'text/html',+ );if (!bodyResult.ok) return rejectWith(bodyResult.reason);return { ok: true, body: bodyResult.body, contentType, finalUrl: target.logicalUrl };}diff --git a/packages/server/src/link-preview/html-metadata.test.ts b/packages/server/src/link-preview/html-metadata.test.tsindex 6a9b135d7..b025f8bd5 100644Binary files a/packages/server/src/link-preview/html-metadata.test.ts and b/packages/server/src/link-preview/html-metadata.test.ts differdiff --git a/packages/server/src/link-preview/html-metadata.ts b/packages/server/src/link-preview/html-metadata.tsindex b0778e353..94782ab20 100644--- a/packages/server/src/link-preview/html-metadata.ts+++ b/packages/server/src/link-preview/html-metadata.ts@@ -284,19 +284,29 @@ function findCloseTag(return null;}+interface HeadScan {+ rawTitle: string;+ metaContent: Map<string, string>;+ faviconHref: string | undefined;+ /**+ * Index just past the head-end boundary (`</head>`'s closing `>`, or the+ * terminator that ends a `<body` open tag's name, mid-tag for an attributed+ * `<body class="a">`), or -1 when the head does not end within `html`.+ */+ headEndOffset: number;+}+/**- * Extract the preview fields from a page's head. Title prefers `og:title` over- * `<title>`; description prefers `og:description` over the `description` meta;- * site name is `og:site_name`. Only content inside the head (before `</head>`- * or `<body>`) counts, so a stray `<title>`/`<meta>` in the body cannot spoof- * the card; a document with no explicit `<head>` (title before `<body>`) still- * yields its head fields. Script/style content and HTML comments are skipped- * wholesale, so markup-shaped strings inside them cannot contribute fields.+ * One context-aware pass over a document head that both collects the metadata+ * fields AND reports where the head ends. `extractHtmlMetadata` (fields) and+ * `findHeadEndOffset` (boundary) are thin views over this single walk, so the+ * streaming fetch and the parser can never disagree on where the head stops.*/-export function extractHtmlMetadata(html: string): RawHtmlMetadata {+function scanHead(html: string): HeadScan {let rawTitle = '';const metaContent = new Map<string, string>();let faviconHref: string | undefined;+ let headEndOffset = -1;try {const lower = asciiLowerCase(html);@@ -334,7 +344,16 @@ export function extractHtmlMetadata(html: string): RawHtmlMetadata {if (next === 0x2f /* / */) {const { name, end } = readTagName(lower, lt + 2);- if (name === 'head') break;+ if (name === 'head') {+ // Record the boundary only once a `>` closes the tag in-buffer. A+ // bare trailing `</head` (end-of-input) is left pending so a chunk+ // boundary cannot mistake a still-growing `</header>` for head-end.+ if (end < len) {+ const gt = lower.indexOf('>', end);+ if (gt !== -1) headEndOffset = gt + 1;+ }+ break;+ }const gt = lower.indexOf('>', end);if (gt === -1) break;i = gt + 1;@@ -349,7 +368,18 @@ export function extractHtmlMetadata(html: string): RawHtmlMetadata {const { name, end: nameEnd } = readTagName(lower, lt + 1);- if (name === 'body') break;+ if (name === 'body') {+ // Head-end at the opening body tag, but only once a real terminator+ // (`>`, `/`, or whitespace) settles the name in-buffer; a bare trailing+ // `<body` waits for the next bytes rather than matching at a boundary.+ if (nameEnd < len) {+ const term = lower.charCodeAt(nameEnd);+ if (term === 0x3e /* > */ || term === 0x2f /* / */ || isWhitespaceCode(term)) {+ headEndOffset = nameEnd + 1;+ }+ }+ break;+ }if (name === 'title') {const contentStart = scanAttributes(html, lower, nameEnd, null);@@ -408,6 +438,21 @@ export function extractHtmlMetadata(html: string): RawHtmlMetadata {// request handler.}+ return { rawTitle, metaContent, faviconHref, headEndOffset };+}++/**+ * Extract the preview fields from a page's head. Title prefers `og:title` over+ * `<title>`; description prefers `og:description` over the `description` meta;+ * site name is `og:site_name`. Only content inside the head (before `</head>`+ * or `<body>`) counts, so a stray `<title>`/`<meta>` in the body cannot spoof+ * the card; a document with no explicit `<head>` (title before `<body>`) still+ * yields its head fields. Script/style content and HTML comments are skipped+ * wholesale, so markup-shaped strings inside them cannot contribute fields.+ */+export function extractHtmlMetadata(html: string): RawHtmlMetadata {+ const { rawTitle, metaContent, faviconHref } = scanHead(html);+const title = sanitizeText(metaContent.get('og:title') ?? rawTitle, MAX_TITLE);const description = sanitizeText(metaContent.get('og:description') ?? metaContent.get('description') ?? '',@@ -423,6 +468,20 @@ export function extractHtmlMetadata(html: string): RawHtmlMetadata {return result;}+/**+ * Byte offset just past the end of a document head (`</head>`'s closing `>`, or+ * the terminator that ends a `<body` open tag's name, mid-tag for an attributed+ * `<body class="a">`), found by the SAME context-aware walk the metadata+ * extractor uses, so markup-shaped bytes inside a head-level script, comment,+ * quoted attribute value, or `<title>` never count as the boundary. Returns -1+ * when the head has not ended within `html`, which lets a streaming caller keep+ * reading; a bare trailing `</head`/`<body` (no terminator yet) stays -1 until+ * the next bytes settle it, so a chunk boundary can never split a false match.+ */+export function findHeadEndOffset(html: string): number {+ return scanHead(html).headEndOffset;+}+/*** The domain shown on the card, derived from the request URL only. `URL`* normalizes IDN hosts to punycode, so a look-alike Unicode host renders as its