Linger is An Open-Source Personal Agent That Runs Fully on Your PC. We aim to break Big Tech’s grip on AI by bringing powerful local AI to everyday computers through efficient model–harness co-design, so everyone can run their own.
import { describe, expect, it, vi } from "vitest";
import { normalizeDocumentContent } from "../../src/document-normalizer/document-normalizer-core.js";
import { WorkerDocumentNormalizer } from "../../src/document-normalizer/document-normalizer.js";
const encode = (value: string): Uint8Array => new TextEncoder().encode(value);
const decode = (value: Uint8Array): string => new TextDecoder().decode(value);
const html = (value: string) => ({
filename: "notes.html",
mediaType: "text/html",
source: encode(value),
});
const convert = (value: string) =>
normalizeDocumentContent(html(value)).then(decode);
describe("HTML document structure", () => {
it("keeps identifiers and addresses together across inline elements", async () => {
await expect(
convert(
"<p>Email: agent@example<span>.com</span>; key: order_<span>42</span>.</p>",
),
).resolves.toBe("Email: [email protected]; key: order_42.\n");
});
it("retains link targets and nested ordered and unordered lists", async () => {
await expect(
convert(
'<p>See <a href="https://example.test/spec?q=a&b=2">the spec</a>.</p><ol><li>First<ul><li>Nested</li></ul></li><li>Second</li></ol>',
),
).resolves.toBe(
"See [the spec](https://example.test/spec?q=a&b=2).\n\n1. First\n - Nested\n2. Second\n",
);
});
it("retains table rows and their column relationships", async () => {
await expect(
convert(
"<table><tr><th>Item</th><th>USD</th></tr><tr><td>Widget</td><td>37.50</td></tr></table>",
),
).resolves.toBe("| Item | USD |\n| --- | --- |\n| Widget | 37.50 |\n");
});
it.each(["", "\uFEFF"])(
"keeps UTF-8 text and a possible BOM despite stale HTML charset metadata",
async (prefix) => {
await expect(
convert(
`${prefix}<meta charset="windows-1252"><p>Caf\u00e9 \u20ac</p>`,
),
).resolves.toBe("Caf\u00e9 \u20ac\n");
},
);
it("preserves indentation and blank lines in a code block", async () => {
await expect(
convert(
"<pre><code> const total = 12;\n\n return total;\n</code></pre>",
),
).resolves.toBe("```\n const total = 12;\n\n return total;\n\n```\n");
});
it("uses the bounded worker executor for HTML while plain text stays in process", async () => {
const execute = vi.fn(async () => encode("worker result\n"));
const normalizer = new WorkerDocumentNormalizer({ execute });
await expect(
normalizer.normalizeDocument(html("<p>Hello</p>")).then(decode),
).resolves.toBe("worker result\n");
expect(execute).toHaveBeenCalledOnce();
await expect(
normalizer
.normalizeDocument({
filename: "notes.txt",
mediaType: "text/plain",
source: encode("Hello"),
})
.then(decode),
).resolves.toBe("Hello\n");
expect(execute).toHaveBeenCalledOnce();
});
it("omits scripts, styles, comments and remote images", async () => {
await expect(
convert(
'<script>throw new Error("script")</script><style>.x{color:red}</style><noscript>Hidden fallback</noscript><!-- comment --><p>Hello & goodbye</p><img src="https://example.invalid/pixel.png">',
),
).resolves.toBe("Hello & goodbye\n");
});
it.each([new Uint8Array([0xff]), encode("<p>bad\u0000text</p>")])(
"keeps the existing text-decoding validation",
async (source) => {
await expect(
normalizeDocumentContent({ ...html(""), source }),
).rejects.toMatchObject({ code: "DOCUMENT_SOURCE_INVALID" });
},
);
it("rejects an unsupported declared charset before worker dispatch", async () => {
const execute = vi.fn(async () => encode("worker result\n"));
const normalizer = new WorkerDocumentNormalizer({ execute });
await expect(
normalizer.normalizeDocument({
...html("<p>ASCII-only body</p>"),
mediaType: "text/html; charset=iso-8859-1",
}),
).rejects.toMatchObject({ code: "DOCUMENT_SOURCE_UNSUPPORTED" });
expect(execute).not.toHaveBeenCalled();
});
it("keeps empty HTML terminal and enforces the Markdown byte limit", async () => {
await expect(convert("<p> </p>")).rejects.toMatchObject({
code: "DOCUMENT_SOURCE_EMPTY",
});
await expect(
normalizeDocumentContent(html("<p>Long text</p>"), 5),
).rejects.toMatchObject({ code: "DOCUMENT_MARKDOWN_TOO_LARGE" });
});
});