Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Release retained source strings in audits and SAM (#593)
Ben Senescu6274e32b6743
diff --git a/src/server/features/sam/samToolOutput.ts b/src/server/features/sam/samToolOutput.tsindex db52ed66..ac862e94 100644--- a/src/server/features/sam/samToolOutput.ts+++ b/src/server/features/sam/samToolOutput.ts@@ -169,7 +169,11 @@ export function capToolOutput(const truncated = note([...trims.values()].filter((trim) => present(copy, trim.path)),);- return typeof copy === "object" && copy !== null && !Array.isArray(copy)- ? { ...copy, truncated }- : { data: copy, truncated };+ // Slicing after the initial clone can retain the original large strings.+ // Detach the finished output before it enters the transcript.+ return structuredClone(+ typeof copy === "object" && copy !== null && !Array.isArray(copy)+ ? { ...copy, truncated }+ : { data: copy, truncated },+ );}diff --git a/src/server/lib/scrape.ts b/src/server/lib/scrape.tsindex dfdded36..d8bd7186 100644--- a/src/server/lib/scrape.ts+++ b/src/server/lib/scrape.ts@@ -150,7 +150,9 @@ async function scrapePage(url: string): Promise<ScrapedPage | null> {if (text.length === 0) {return null;}- return { url, title: extractTitle(html), text };+ // Extracted titles and shortened text can retain their entire source string+ // in V8. Detach them before pages accumulate in results or chat transcripts.+ return structuredClone({ url, title: extractTitle(html), text });}/**diff --git a/src/server/lib/string-retention.test.ts b/src/server/lib/string-retention.test.tsnew file mode 100644index 00000000..9abc6ac2--- /dev/null+++ b/src/server/lib/string-retention.test.ts@@ -0,0 +1,88 @@+import { spawnSync } from "node:child_process";+import { describe, expect, it } from "vitest";++describe("retained string memory", () => {+ it.each([+ "scraped title",+ "scraped text",+ "tool output",+ "tool output with smaller budget",+ ])(+ "releases the large source behind %s",+ (scenario) => {+ // Isolate GC from Vitest's heap. Each result must outlive its independently+ // allocated source, just as queued pages and transcript entries do.+ const result = spawnSync(+ process.execPath,+ [+ "--expose-gc",+ "--import",+ "tsx",+ "--input-type=module",+ "-e",+ `+ import assert from "node:assert/strict";+ import { readPages } from ${JSON.stringify(new URL("./scrape.ts", import.meta.url).href)};+ import { capToolOutput } from ${JSON.stringify(new URL("../features/sam/samToolOutput.ts", import.meta.url).href)};+ const scenario = ${JSON.stringify(scenario)};+ const title = "Example memory regression 🌱";+ globalThis.fetch = async () => {+ const content = scenario === "scraped text"+ ? "sample words ".repeat(100_000)+ : "<script>" + "x".repeat(1_500_000) + "</script>Visible text";+ const bytes = new TextEncoder().encode(+ "<title>" + title + "</title><body>" + content + "</body>");+ let offset = 0;+ return new Response(new ReadableStream({+ pull(controller) {+ if (offset >= bytes.length) return controller.close();+ controller.enqueue(bytes.subarray(offset, offset + 65536));+ offset += 65536;+ },+ }), { headers: { "content-type": "text/html" } });+ };+ const makeResult = async () => {+ if (scenario.startsWith("scraped")) {+ // Documentation-only IP avoids DNS; fetch is fully stubbed.+ const result = await readPages(["https://192.0.2.1/example"]);+ assert.equal(result.blocked, false);+ assert.equal(result.pages[0].title, title);+ assert.ok(result.pages[0].text.length <= 4000);+ return result;+ }+ const input = { pages: [{ text: new TextDecoder().decode(+ new TextEncoder().encode("x".repeat(1_500_000))) }] };+ const budget = scenario.endsWith("smaller budget") ? 2000 : 32000;+ const result = capToolOutput(input, budget);+ assert.ok(JSON.stringify(result).length <= budget);+ assert.equal(input.pages[0].text.length, 1_500_000);+ assert.ok(result.pages[0].text.length <= 8000);+ assert.ok(result.truncated.includes("shortened"));+ return result;+ };+ const collect = async () => {+ await new Promise(resolve => setTimeout(resolve, 0));+ globalThis.gc();+ globalThis.gc();+ const { heapUsed, external } = process.memoryUsage();+ return heapUsed + external;+ };+ await makeResult();+ const before = await collect();+ const results = [];+ for (let i = 0; i < 40; i++) results.push(await makeResult());+ const retained = (await collect()) - before;+ assert.equal(results.length, 40);+ assert.ok(retained < 16 * 1024 * 1024,+ scenario + " retained " + retained + " bytes of source strings");+ `,+ ],+ { encoding: "utf8", timeout: 20_000 },+ );++ expect(result.error).toBeUndefined();+ expect(result.status, result.stderr).toBe(0);+ },+ 25_000,+ );+});diff --git a/src/server/workflows/site-audit-workflow-helpers.test.ts b/src/server/workflows/site-audit-workflow-helpers.test.tsnew file mode 100644index 00000000..1943a758--- /dev/null+++ b/src/server/workflows/site-audit-workflow-helpers.test.ts@@ -0,0 +1,147 @@+import { spawnSync } from "node:child_process";+import { afterEach, describe, expect, it, vi } from "vitest";+import { crawlPage } from "./site-audit-workflow-helpers";++afterEach(() => vi.unstubAllGlobals());++describe("crawlPage", () => {+ it("preserves extracted metadata and nested values when releasing the HTML", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue(+ new Response(+ `<html><head>+ <title>Example & café 🌱</title>+ <meta name="description" content="Example description & more">+ <meta name="robots" content="noindex">+ <meta property="og:title" content="Example social title 🌱">+ <meta property="og:description" content="Example description">+ <meta property="og:image" content="/social.png">+ <link rel="canonical" href="/canonical">+ <link rel="alternate" hreflang="en" href="/en">+ <script type="application/ld+json">{}</script>+ </head><body><h1>Hello</h1><h2>World</h2>+ <img src="/photo.png" alt="Example photo 🌱"><img src="/missing.png">+ <img alt="">+ <a href="/next" rel="nofollow">More & more 🌱</a>+ <a href="https://other.example/">External</a>+ </body></html>`,+ {+ headers: {+ "content-type": "text/html",+ "x-robots-tag": "nofollow",+ link: '<https://example.com/header>; rel="canonical"',+ },+ },+ ),+ ),+ );++ const page = await crawlPage("https://example.com/", 2, true);++ expect(page).toMatchObject({+ url: "https://example.com/",+ statusCode: 200,+ fetchClass: "ok",+ redirectUrl: null,+ title: "Example & café 🌱",+ metaDescription: "Example description & more",+ canonicalUrl: "https://example.com/canonical",+ robotsMeta: "noindex",+ xRobotsTag: "nofollow",+ headerCanonicalUrl: "https://example.com/header",+ ogTitle: "Example social title 🌱",+ ogDescription: "Example description",+ ogImage: "/social.png",+ h1Count: 1,+ h2Count: 1,+ h3Count: 0,+ headingOrder: [1, 2],+ isHtml: true,+ imagesTotal: 3,+ imagesMissingAlt: 1,+ images: [+ { src: "/photo.png", alt: "Example photo 🌱" },+ { src: "/missing.png", alt: null },+ { src: null, alt: "" },+ ],+ links: [+ {+ targetUrl: "https://example.com/next",+ anchor: "More & more 🌱",+ isInternal: true,+ isNofollow: true,+ },+ {+ targetUrl: "https://other.example/",+ anchor: "External",+ isInternal: false,+ isNofollow: false,+ },+ ],+ hasStructuredData: true,+ hreflangTags: ["en"],+ isIndexable: false,+ crawlDepth: 2,+ inSitemap: true,+ });+ expect(page.contentHash).toMatch(/^[a-f0-9]{64}$/);+ expect(page.htmlBytes).toBeGreaterThan(0);+ });++ it("does not retain large source HTML in queued crawl results", () => {+ // A dedicated V8 process makes GC available without depending on the test+ // runner's heap. Exercise the real reader/parser with separate streamed+ // bodies; keeping just their titles used to retain every 1 MiB document.+ const result = spawnSync(+ process.execPath,+ [+ "--expose-gc",+ "--import",+ "tsx",+ "--input-type=module",+ "-e",+ `+ import assert from "node:assert/strict";+ import { crawlPage } from ${JSON.stringify(new URL("./site-audit-workflow-helpers.ts", import.meta.url).href)};+ globalThis.fetch = async () => {+ const html = '<title>Example memory regression 🌱</title>' ++ '<meta name="description" content="A small description">' ++ '<script>' + 'x'.repeat(1024 * 1024) + '</script>';+ const bytes = new TextEncoder().encode(html);+ let offset = 0;+ return new Response(new ReadableStream({+ pull(controller) {+ if (offset >= bytes.length) return controller.close();+ controller.enqueue(bytes.subarray(offset, offset + 65536));+ offset += 65536;+ },+ }), { headers: { "content-type": "text/html" } });+ };+ const collect = async () => {+ await new Promise(resolve => setTimeout(resolve, 0));+ globalThis.gc();+ globalThis.gc();+ const { heapUsed, external } = process.memoryUsage();+ return heapUsed + external;+ };+ await crawlPage("https://example.com/warmup", 0, true);+ const before = await collect();+ const pages = [];+ for (let i = 0; i < 50; i++) {+ pages.push(await crawlPage("https://example.com/" + i, 0, true));+ }+ const retained = (await collect()) - before;+ assert.equal(pages.length, 50);+ assert.equal(pages[49].title, "Example memory regression 🌱");+ assert.ok(retained < 16 * 1024 * 1024,+ "Queued results retained " + retained + " bytes of source HTML");+ `,+ ],+ { encoding: "utf8", timeout: 20_000 },+ );++ expect(result.error).toBeUndefined();+ expect(result.status, result.stderr).toBe(0);+ }, 25_000);+});diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.tsindex a03463f3..9aebba90 100644--- a/src/server/workflows/site-audit-workflow-helpers.ts+++ b/src/server/workflows/site-audit-workflow-helpers.ts@@ -144,7 +144,10 @@ export async function crawlPage(const headingCount = (level: number) =>analysis.headingOrder.filter((h) => h === level).length;- return {+ // Parser strings can be V8 slices backed by the entire HTML body. Detach+ // the finished result before persistence queues retain it: otherwise a+ // few KB of metadata can keep ~2 MiB of decoded HTML alive per page.+ return structuredClone({id: crypto.randomUUID(),url,statusCode,@@ -187,7 +190,7 @@ export async function crawlPage(responseTimeMs,crawlDepth,inSitemap,- };+ });} catch (error) {const responseTimeMs = Date.now() - startTime;console.warn(`Failed to crawl ${url}:`, error);