Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
/**
* Parity tests for the streaming (htmlparser2) page analyzer against a
* cheerio/DOM reference implementation — the exact logic the analyzer
* replaced. Cheerio stays as a devDependency for this test only.
*/
import * as cheerio from "cheerio";
import { describe, expect, it } from "vitest";
import { analyzeHtml } from "@/server/lib/audit/page-analyzer";
import { normalizeUrl, isSameOrigin } from "@/server/lib/audit/url-utils";
import type { PageAnalysis, PageLink } from "@/server/lib/audit/types";
/**
* The previous cheerio implementation (minus passthrough fields), plus
* separators around block elements so minified markup keeps word boundaries.
*/
function analyzeHtmlWithCheerio(html: string, pageUrl: string): PageAnalysis {
const $ = cheerio.load(html);
const title = $("title").first().text().trim();
const metaDescription =
$('meta[name="description"]').first().attr("content")?.trim() ?? "";
const canonical = $('link[rel="canonical"]').first().attr("href") ?? null;
const robotsMeta = $('meta[name="robots"]').first().attr("content") ?? null;
const ogTitle =
$('meta[property="og:title"]').first().attr("content") ?? null;
const ogDescription =
$('meta[property="og:description"]').first().attr("content") ?? null;
const ogImage =
$('meta[property="og:image"]').first().attr("content") ?? null;
const h1s: string[] = [];
$("h1").each((_, el) => {
h1s.push($(el).text().trim());
});
const headingOrder: number[] = [];
$("h1, h2, h3, h4, h5, h6").each((_, el) => {
const tag =
"tagName" in el && typeof el.tagName === "string"
? el.tagName.toLowerCase()
: null;
if (tag) {
const level = parseInt(tag.charAt(1), 10);
if (!isNaN(level)) headingOrder.push(level);
}
});
const bodyClone = $("body").clone();
bodyClone.find("script, style, noscript, svg").remove();
bodyClone
.find(
"address,article,aside,blockquote,br,caption,dd,details,dialog,div,dl,dt,fieldset,figcaption,figure,footer,form,h1,h2,h3,h4,h5,h6,header,hgroup,hr,li,main,nav,ol,p,pre,section,summary,table,td,th,tr,ul",
)
.before(" ")
.after(" ");
const bodyText = bodyClone.text().replace(/\s+/g, " ").trim();
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
const images: Array<{ src: string | null; alt: string | null }> = [];
$("img").each((_, el) => {
images.push({
src: $(el).attr("src") ?? null,
alt: $(el).attr("alt") ?? null,
});
});
const linksByTarget = new Map<string, PageLink>();
$("a[href]").each((_, el) => {
const href = $(el).attr("href");
if (!href) return;
if (/^(javascript:|mailto:|tel:|#)/.test(href)) return;
const resolved = normalizeUrl(href, pageUrl);
if (!resolved) return;
if (linksByTarget.has(resolved)) return;
const anchor = $(el).text().replace(/\s+/g, " ").trim().slice(0, 200);
const rel = $(el).attr("rel")?.toLowerCase() ?? "";
linksByTarget.set(resolved, {
targetUrl: resolved,
anchor: anchor || null,
isInternal: isSameOrigin(resolved, pageUrl),
isNofollow: rel.split(/\s+/).includes("nofollow"),
});
});
let hasStructuredData = false;
$('script[type="application/ld+json"]').each(() => {
hasStructuredData = true;
});
const hreflangTags: string[] = [];
$('link[rel="alternate"][hreflang]').each((_, el) => {
const hreflang = $(el).attr("hreflang");
if (hreflang) hreflangTags.push(hreflang);
});
return {
url: pageUrl,
statusCode: 200,
redirectUrl: null,
responseTimeMs: 0,
title,
metaDescription,
canonical,
robotsMeta,
ogTitle,
ogDescription,
ogImage,
h1s,
headingOrder,
wordCount,
bodyText,
images,
links: Array.from(linksByTarget.values()),
hasStructuredData,
hreflangTags,
};
}
const PAGE_URL = "https://example.com/blog/post";
function expectParity(html: string) {
const streamed = analyzeHtml(html, PAGE_URL, 200, 0);
const reference = analyzeHtmlWithCheerio(html, PAGE_URL);
const { javascriptShell: _javascriptShell, ...existingAnalysis } = streamed;
expect(existingAnalysis).toEqual(reference);
}
describe("analyzeHtml parity with the DOM reference", () => {
it("matches on a full, well-formed document", () => {
expectParity(`<!DOCTYPE html>
<html><head>
<title> The Title </title>
<meta name="description" content=" A description. ">
<meta name="robots" content="index, follow">
<meta property="og:title" content="OG Title">
<meta property="og:description" content="OG Desc">
<meta property="og:image" content="/og.png">
<link rel="canonical" href="https://example.com/blog/post">
<link rel="alternate" hreflang="en" href="/en">
<link rel="alternate" hreflang="de" href="/de">
<script type="application/ld+json">{"@type":"Article"}</script>
</head><body>
<h1>Main <em>Heading</em></h1>
<h2>Sub</h2><h3>Deep</h3><h2>Sub 2</h2>
<p>Some visible body text with <strong>bold words</strong> here.</p>
<script>ignored();</script>
<style>.x{}</style>
<svg><title>icon</title><text>svg text</text></svg>
<noscript><img src="/ns.png"><p>noscript text</p></noscript>
<img src="/a.png" alt="A picture">
<img src="/b.png" alt="">
<img src="/c.png">
<img alt="no src">
<a href="/relative">Relative <span>link</span></a>
<a href="https://example.com/relative">Duplicate target</a>
<a href="https://other.example/x" rel="NoFollow sponsored">External</a>
<a href="mailto:[email protected]">Mail</a>
<a href="#frag">Fragment</a>
<a href="javascript:void(0)">JS</a>
<a href="/empty-anchor"><img src="/img-link.png" alt=""></a>
</body></html>`);
});
it("matches on documents with no head, body, or title", () => {
expectParity(
`<h1>Bare fragment</h1><p>Just text and a <a href="/x">link</a>.</p>`,
);
});
it("matches on an empty document", () => {
expectParity("");
});
it("matches on a document with only a head", () => {
expectParity(
`<html><head><title>Head only</title><meta name="description" content="d"></head></html>`,
);
});
it("matches with duplicate metas and titles (first wins)", () => {
expectParity(`<html><head>
<title>First</title><title>Second</title>
<meta name="description" content="first desc">
<meta name="description" content="second desc">
<link rel="canonical" href="/first"><link rel="canonical" href="/second">
</head><body><p>text</p></body></html>`);
});
it("matches on unclosed and misnested tags", () => {
expectParity(`<html><body>
<h1>Unclosed heading
<p>Paragraph <b>bold <i>both</b> italic?</i>
<a href="/one">first <a href="/two">second</a>
<div>trailing text`);
});
it("matches on entity-heavy content", () => {
expectParity(`<html><head><title>A & B <C></title></head>
<body><h1>Café "menu"</h1>
<p>1 < 2 && 3 > 2</p>
<a href="/x?a=1&b=2">Query & anchor</a></body></html>`);
});
it("matches heading order across nesting", () => {
expectParity(`<body><h3>three</h3><div><h1>one</h1><section><h2>two</h2>
<h6>six</h6></section></div><h4>four</h4></body>`);
});
it("matches word counts with whitespace-heavy markup", () => {
expectParity(`<body>
<p>
one
two three
</p>
<ul><li>four</li><li>five</li></ul>
</body>`);
});
});
describe("analyzeHtml word count", () => {
const analyzeBody = (body: string) =>
analyzeHtml(`<body>${body}</body>`, PAGE_URL, 200, 0);
it.each([
["paragraphs", "<p>one</p><p>two</p>"],
["table cells", "<table><tr><td>one</td><td>two</td></tr></table>"],
["line breaks", "one<br>two"],
])("keeps word boundaries between %s in minified HTML", (_label, body) => {
expect(analyzeBody(body).bodyText).toBe("one two");
});
it("does not split words across inline markup", () => {
expect(analyzeBody("<p>Open<strong>SEO</strong></p>").wordCount).toBe(1);
});
it("counts words in scripts written without spaces", () => {
expect(
analyzeBody("<p>吉隆坡市中心永久地契公寓投资指南</p>").wordCount,
).toBe(8);
expect(analyzeBody("<p>東京の不動産投資ガイド</p>").wordCount).toBe(5);
expect(analyzeBody("<p>KLCC 公寓指南 2026</p>").wordCount).toBe(4);
});
});
describe("analyzeHtml meta names are case-insensitive", () => {
const analyzeHead = (head: string) =>
analyzeHtml(`<head>${head}</head>`, PAGE_URL, 200, 0);
it("reads the first description regardless of name case", () => {
expect(
analyzeHead(
`<meta name="Description" content="first"><meta name="description" content="second">`,
).metaDescription,
).toBe("first");
});
it("reads robots for name=ROBOTS", () => {
expect(
analyzeHead(`<meta name="ROBOTS" content="noindex">`).robotsMeta,
).toBe("noindex");
});
});
describe("analyzeHtml extraction caps", () => {
it("caps links and images per page", () => {
const links = Array.from(
{ length: 1_100 },
(_, i) => `<a href="/p/${i}">link ${i}</a>`,
).join("");
const images = Array.from(
{ length: 1_100 },
(_, i) => `<img src="/i/${i}.png">`,
).join("");
const analysis = analyzeHtml(
`<body>${links}${images}</body>`,
"https://example.com/",
200,
100,
);
expect(analysis.links).toHaveLength(1_000);
expect(analysis.images).toHaveLength(1_000);
});
});
describe("JavaScript app-shell detection", () => {
it.each(["root", "app", "__next", "__nuxt"])(
"recognizes an empty %s app shell",
(id) => {
expect(
analyzeHtml(
`<body><div id="${id}">Loading...</div><script src="/app.js"></script></body>`,
PAGE_URL,
200,
0,
).javascriptShell,
).toBe(true);
},
);
it("recognizes Angular's app element", () => {
expect(
analyzeHtml(
'<app-root></app-root><script type="module" src="/main.js"></script>',
PAGE_URL,
200,
0,
).javascriptShell,
).toBe(true);
});
it.each([
'<div id="root"><h2>A server-rendered heading</h2></div><script src="/app.js"></script>',
'<div id="root"><a href="/about">About</a></div><script src="/app.js"></script>',
'<div id="app"><img src="/photo.png"></div><script src="/app.js"></script>',
'<div id="root"></div><script type="application/ld+json">{}</script>',
'<p>Short but valid content</p><script src="/analytics.js"></script>',
'<div id="root"></div>',
`<div id="root">${"Readable server-rendered content ".repeat(10)}</div><script src="/app.js"></script>`,
])(
"does not confuse ordinary HTML, hydrated content, or structured data with a shell",
(html) => {
expect(analyzeHtml(html, PAGE_URL, 200, 0).javascriptShell).toBe(false);
},
);
});