Skip to content

Open source alternative to Semrush and Ahrefs

Release retained source strings in audits and SAM (#593)

Ben Senescu6274e32b6743
diff --git a/src/server/features/sam/samToolOutput.ts b/src/server/features/sam/samToolOutput.ts
index db52ed66..ac862e94 100644
--- a/src/server/features/sam/samToolOutput.ts
+++ b/src/server/features/sam/samToolOutput.ts
@@ -169,7 +169,11 @@ export function capToolOutput(
const truncated = note(
[...trims.values()].filter((trim) => present(copy, trim.path)),
);
- return typeof copy === "object" && copy !== null && !Array.isArray(copy)
- ? { ...copy, truncated }
- : { data: copy, truncated };
+ // Slicing after the initial clone can retain the original large strings.
+ // Detach the finished output before it enters the transcript.
+ return structuredClone(
+ typeof copy === "object" && copy !== null && !Array.isArray(copy)
+ ? { ...copy, truncated }
+ : { data: copy, truncated },
+ );
}
diff --git a/src/server/lib/scrape.ts b/src/server/lib/scrape.ts
index dfdded36..d8bd7186 100644
--- a/src/server/lib/scrape.ts
+++ b/src/server/lib/scrape.ts
@@ -150,7 +150,9 @@ async function scrapePage(url: string): Promise<ScrapedPage | null> {
if (text.length === 0) {
return null;
}
- return { url, title: extractTitle(html), text };
+ // Extracted titles and shortened text can retain their entire source string
+ // in V8. Detach them before pages accumulate in results or chat transcripts.
+ return structuredClone({ url, title: extractTitle(html), text });
}
/**
diff --git a/src/server/lib/string-retention.test.ts b/src/server/lib/string-retention.test.ts
new file mode 100644
index 00000000..9abc6ac2
--- /dev/null
+++ b/src/server/lib/string-retention.test.ts
@@ -0,0 +1,88 @@
+import { spawnSync } from "node:child_process";
+import { describe, expect, it } from "vitest";
+
+describe("retained string memory", () => {
+ it.each([
+ "scraped title",
+ "scraped text",
+ "tool output",
+ "tool output with smaller budget",
+ ])(
+ "releases the large source behind %s",
+ (scenario) => {
+ // Isolate GC from Vitest's heap. Each result must outlive its independently
+ // allocated source, just as queued pages and transcript entries do.
+ const result = spawnSync(
+ process.execPath,
+ [
+ "--expose-gc",
+ "--import",
+ "tsx",
+ "--input-type=module",
+ "-e",
+ `
+ import assert from "node:assert/strict";
+ import { readPages } from ${JSON.stringify(new URL("./scrape.ts", import.meta.url).href)};
+ import { capToolOutput } from ${JSON.stringify(new URL("../features/sam/samToolOutput.ts", import.meta.url).href)};
+ const scenario = ${JSON.stringify(scenario)};
+ const title = "Example memory regression 🌱";
+ globalThis.fetch = async () => {
+ const content = scenario === "scraped text"
+ ? "sample words ".repeat(100_000)
+ : "<script>" + "x".repeat(1_500_000) + "</script>Visible text";
+ const bytes = new TextEncoder().encode(
+ "<title>" + title + "</title><body>" + content + "</body>");
+ let offset = 0;
+ return new Response(new ReadableStream({
+ pull(controller) {
+ if (offset >= bytes.length) return controller.close();
+ controller.enqueue(bytes.subarray(offset, offset + 65536));
+ offset += 65536;
+ },
+ }), { headers: { "content-type": "text/html" } });
+ };
+ const makeResult = async () => {
+ if (scenario.startsWith("scraped")) {
+ // Documentation-only IP avoids DNS; fetch is fully stubbed.
+ const result = await readPages(["https://192.0.2.1/example"]);
+ assert.equal(result.blocked, false);
+ assert.equal(result.pages[0].title, title);
+ assert.ok(result.pages[0].text.length <= 4000);
+ return result;
+ }
+ const input = { pages: [{ text: new TextDecoder().decode(
+ new TextEncoder().encode("x".repeat(1_500_000))) }] };
+ const budget = scenario.endsWith("smaller budget") ? 2000 : 32000;
+ const result = capToolOutput(input, budget);
+ assert.ok(JSON.stringify(result).length <= budget);
+ assert.equal(input.pages[0].text.length, 1_500_000);
+ assert.ok(result.pages[0].text.length <= 8000);
+ assert.ok(result.truncated.includes("shortened"));
+ return result;
+ };
+ const collect = async () => {
+ await new Promise(resolve => setTimeout(resolve, 0));
+ globalThis.gc();
+ globalThis.gc();
+ const { heapUsed, external } = process.memoryUsage();
+ return heapUsed + external;
+ };
+ await makeResult();
+ const before = await collect();
+ const results = [];
+ for (let i = 0; i < 40; i++) results.push(await makeResult());
+ const retained = (await collect()) - before;
+ assert.equal(results.length, 40);
+ assert.ok(retained < 16 * 1024 * 1024,
+ scenario + " retained " + retained + " bytes of source strings");
+ `,
+ ],
+ { encoding: "utf8", timeout: 20_000 },
+ );
+
+ expect(result.error).toBeUndefined();
+ expect(result.status, result.stderr).toBe(0);
+ },
+ 25_000,
+ );
+});
diff --git a/src/server/workflows/site-audit-workflow-helpers.test.ts b/src/server/workflows/site-audit-workflow-helpers.test.ts
new file mode 100644
index 00000000..1943a758
--- /dev/null
+++ b/src/server/workflows/site-audit-workflow-helpers.test.ts
@@ -0,0 +1,147 @@
+import { spawnSync } from "node:child_process";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { crawlPage } from "./site-audit-workflow-helpers";
+
+afterEach(() => vi.unstubAllGlobals());
+
+describe("crawlPage", () => {
+ it("preserves extracted metadata and nested values when releasing the HTML", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue(
+ new Response(
+ `<html><head>
+ <title>Example &amp; café 🌱</title>
+ <meta name="description" content="Example description &amp; more">
+ <meta name="robots" content="noindex">
+ <meta property="og:title" content="Example social title 🌱">
+ <meta property="og:description" content="Example description">
+ <meta property="og:image" content="/social.png">
+ <link rel="canonical" href="/canonical">
+ <link rel="alternate" hreflang="en" href="/en">
+ <script type="application/ld+json">{}</script>
+ </head><body><h1>Hello</h1><h2>World</h2>
+ <img src="/photo.png" alt="Example photo 🌱"><img src="/missing.png">
+ <img alt="">
+ <a href="/next" rel="nofollow">More &amp; more 🌱</a>
+ <a href="https://other.example/">External</a>
+ </body></html>`,
+ {
+ headers: {
+ "content-type": "text/html",
+ "x-robots-tag": "nofollow",
+ link: '<https://example.com/header>; rel="canonical"',
+ },
+ },
+ ),
+ ),
+ );
+
+ const page = await crawlPage("https://example.com/", 2, true);
+
+ expect(page).toMatchObject({
+ url: "https://example.com/",
+ statusCode: 200,
+ fetchClass: "ok",
+ redirectUrl: null,
+ title: "Example & café 🌱",
+ metaDescription: "Example description & more",
+ canonicalUrl: "https://example.com/canonical",
+ robotsMeta: "noindex",
+ xRobotsTag: "nofollow",
+ headerCanonicalUrl: "https://example.com/header",
+ ogTitle: "Example social title 🌱",
+ ogDescription: "Example description",
+ ogImage: "/social.png",
+ h1Count: 1,
+ h2Count: 1,
+ h3Count: 0,
+ headingOrder: [1, 2],
+ isHtml: true,
+ imagesTotal: 3,
+ imagesMissingAlt: 1,
+ images: [
+ { src: "/photo.png", alt: "Example photo 🌱" },
+ { src: "/missing.png", alt: null },
+ { src: null, alt: "" },
+ ],
+ links: [
+ {
+ targetUrl: "https://example.com/next",
+ anchor: "More & more 🌱",
+ isInternal: true,
+ isNofollow: true,
+ },
+ {
+ targetUrl: "https://other.example/",
+ anchor: "External",
+ isInternal: false,
+ isNofollow: false,
+ },
+ ],
+ hasStructuredData: true,
+ hreflangTags: ["en"],
+ isIndexable: false,
+ crawlDepth: 2,
+ inSitemap: true,
+ });
+ expect(page.contentHash).toMatch(/^[a-f0-9]{64}$/);
+ expect(page.htmlBytes).toBeGreaterThan(0);
+ });
+
+ it("does not retain large source HTML in queued crawl results", () => {
+ // A dedicated V8 process makes GC available without depending on the test
+ // runner's heap. Exercise the real reader/parser with separate streamed
+ // bodies; keeping just their titles used to retain every 1 MiB document.
+ const result = spawnSync(
+ process.execPath,
+ [
+ "--expose-gc",
+ "--import",
+ "tsx",
+ "--input-type=module",
+ "-e",
+ `
+ import assert from "node:assert/strict";
+ import { crawlPage } from ${JSON.stringify(new URL("./site-audit-workflow-helpers.ts", import.meta.url).href)};
+ globalThis.fetch = async () => {
+ const html = '<title>Example memory regression 🌱</title>' +
+ '<meta name="description" content="A small description">' +
+ '<script>' + 'x'.repeat(1024 * 1024) + '</script>';
+ const bytes = new TextEncoder().encode(html);
+ let offset = 0;
+ return new Response(new ReadableStream({
+ pull(controller) {
+ if (offset >= bytes.length) return controller.close();
+ controller.enqueue(bytes.subarray(offset, offset + 65536));
+ offset += 65536;
+ },
+ }), { headers: { "content-type": "text/html" } });
+ };
+ const collect = async () => {
+ await new Promise(resolve => setTimeout(resolve, 0));
+ globalThis.gc();
+ globalThis.gc();
+ const { heapUsed, external } = process.memoryUsage();
+ return heapUsed + external;
+ };
+ await crawlPage("https://example.com/warmup", 0, true);
+ const before = await collect();
+ const pages = [];
+ for (let i = 0; i < 50; i++) {
+ pages.push(await crawlPage("https://example.com/" + i, 0, true));
+ }
+ const retained = (await collect()) - before;
+ assert.equal(pages.length, 50);
+ assert.equal(pages[49].title, "Example memory regression 🌱");
+ assert.ok(retained < 16 * 1024 * 1024,
+ "Queued results retained " + retained + " bytes of source HTML");
+ `,
+ ],
+ { encoding: "utf8", timeout: 20_000 },
+ );
+
+ expect(result.error).toBeUndefined();
+ expect(result.status, result.stderr).toBe(0);
+ }, 25_000);
+});
diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.ts
index a03463f3..9aebba90 100644
--- a/src/server/workflows/site-audit-workflow-helpers.ts
+++ b/src/server/workflows/site-audit-workflow-helpers.ts
@@ -144,7 +144,10 @@ export async function crawlPage(
const headingCount = (level: number) =>
analysis.headingOrder.filter((h) => h === level).length;
- return {
+ // Parser strings can be V8 slices backed by the entire HTML body. Detach
+ // the finished result before persistence queues retain it: otherwise a
+ // few KB of metadata can keep ~2 MiB of decoded HTML alive per page.
+ return structuredClone({
id: crypto.randomUUID(),
url,
statusCode,
@@ -187,7 +190,7 @@ export async function crawlPage(
responseTimeMs,
crawlDepth,
inSitemap,
- };
+ });
} catch (error) {
const responseTimeMs = Date.now() - startTime;
console.warn(`Failed to crawl ${url}:`, error);