Linger is An Open-Source Personal Agent That Runs Fully on Your PC. We aim to break Big Tech’s grip on AI by bringing powerful local AI to everyday computers through efficient model–harness co-design, so everyone can run their own.
import { readFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it, vi } from "vitest";
import {
canonicalizeMarkdown,
isInProcessDocumentSource,
normalizeDocumentContent,
} from "../../src/document-normalizer/document-normalizer-core.js";
import {
DOC_MEDIA_TYPE,
DocumentNormalizationError,
DOCX_MEDIA_TYPE,
GZIP_MEDIA_TYPE,
MAX_DOCUMENT_MARKDOWN_BYTES,
MAX_DOCUMENT_SOURCE_BYTES,
PDF_MEDIA_TYPE,
PNG_MEDIA_TYPE,
SVG_MEDIA_TYPE,
WorkerDocumentNormalizer,
XLSX_MEDIA_TYPE,
ZIP_MEDIA_TYPE,
} from "../../src/document-normalizer/document-normalizer.js";
import {
docxFixture,
gzipFixture,
pdfFixture,
pngFixture,
pngWithTextFixture,
svgFixture,
xlsxFixture,
zipFixture,
} from "./fixtures/document-normalizer-fixtures.js";
const decode = (value: Uint8Array): string => new TextDecoder().decode(value);
const encode = (value: string): Uint8Array => new TextEncoder().encode(value);
const sampleDocPath = join(
dirname(fileURLToPath(import.meta.url)),
"fixtures/sample.doc",
);
describe("document normalization", () => {
it("extracts a PDF text layer as canonical Markdown", async () => {
const markdown = decode(
await normalizeDocumentContent({
mediaType: PDF_MEDIA_TYPE,
source: pdfFixture(),
}),
);
expect(markdown).toBe("PDF text layer\n");
});
it("preserves DOCX formatting and applies NFC", async () => {
const markdown = decode(
await normalizeDocumentContent({
mediaType: DOCX_MEDIA_TYPE,
source: docxFixture(),
}),
);
expect(markdown).toBe("**Caf\u00e9**\n\nSecond line\n");
expect(markdown).toBe(markdown.normalize("NFC"));
});
it("extracts body text from a legacy Word DOC", async () => {
const markdown = decode(
await normalizeDocumentContent({
filename: "sample.doc",
mediaType: DOC_MEDIA_TYPE,
source: await readFile(sampleDocPath),
}),
);
expect(markdown).toContain("SimplexAI Office Preview");
expect(markdown).toContain("Aperçu multilingue");
});
it("resolves a DOC from filename when the stored media type is generic", async () => {
const markdown = decode(
await normalizeDocumentContent({
filename: "notes.doc",
mediaType: "application/octet-stream",
representationFormat: "doc",
source: await readFile(sampleDocPath),
}),
);
expect(markdown).toContain("SimplexAI Office Preview");
});
it("converts HTML, CSV, JSON, and XLSX into Markdown", async () => {
await expect(
normalizeDocumentContent({
mediaType: "text/html; charset=utf-8",
source: encode("<h1>Title</h1><p>Hello <b>world</b></p>"),
}).then(decode),
).resolves.toBe("# Title\n\nHello **world**\n");
await expect(
normalizeDocumentContent({
filename: "people.csv",
mediaType: "text/csv",
source: encode("name,count\nAda,42\n"),
}).then(decode),
).resolves.toBe("| name | count |\n| --- | --- |\n| Ada | 42 |\n");
await expect(
normalizeDocumentContent({
mediaType: "application/json",
source: encode('{"name":"Ada"}'),
}).then(decode),
).resolves.toBe('```json\n{\n "name": "Ada"\n}\n```\n');
await expect(
normalizeDocumentContent({
mediaType: XLSX_MEDIA_TYPE,
source: xlsxFixture(),
}).then(decode),
).resolves.toBe(
"## People\n\n| Name | Count |\n| --- | --- |\n| Ada | 42 |\n",
);
});
it.each([
'text/csv; header=present; charset="UTF-8"',
'application/octet-stream; charset="utf8"',
'application/vnd.example.table; profile="https://example.test/schema; charset=windows-1252"; charset=utf-8',
])("normalizes UTF-8 CSV with MIME parameters: %s", async (mediaType) => {
const input = {
filename: "people.csv",
mediaType,
source: encode("name,count\nCaf\u00e9,42\n"),
};
expect(isInProcessDocumentSource(input)).toBe(true);
await expect(normalizeDocumentContent(input).then(decode)).resolves.toBe(
"| name | count |\n| --- | --- |\n| Caf\u00e9 | 42 |\n",
);
});
it.each([
'text/csv; header=present; charset="windows-1252"',
'application/octet-stream; charset="UTF-16"',
])("still rejects unsupported declared encodings: %s", async (mediaType) => {
await expect(
normalizeDocumentContent({
filename: "people.csv",
mediaType,
source: encode("name,count\nAda,42\n"),
}),
).rejects.toMatchObject({ code: "DOCUMENT_SOURCE_UNSUPPORTED" });
});
it("still validates text bytes when an accepted charset is quoted", async () => {
await expect(
normalizeDocumentContent({
filename: "notes.txt",
mediaType: 'text/plain; charset="utf-8"',
source: new Uint8Array([0x63, 0x61, 0x66, 0xe9]),
}),
).rejects.toMatchObject({ code: "DOCUMENT_SOURCE_INVALID" });
});
it("extracts archive and SVG text and reads no raster image", async () => {
await expect(
normalizeDocumentContent({
filename: "bundle.zip",
mediaType: ZIP_MEDIA_TYPE,
source: zipFixture([
{ name: "notes.md", value: "# From zip\n" },
{ name: ".DS_Store", value: "junk" },
{
name: "photo.png",
value: pngWithTextFixture("Comment", "Album cover"),
},
]),
}).then(decode),
).resolves.toBe("## notes.md\n\n# From zip\n");
await expect(
normalizeDocumentContent({
filename: "notes.md.gz",
mediaType: GZIP_MEDIA_TYPE,
source: gzipFixture(encode("# From gzip\n")),
}).then(decode),
).resolves.toBe("# From gzip\n");
await expect(
normalizeDocumentContent({
filename: "icon.svg",
mediaType: SVG_MEDIA_TYPE,
source: svgFixture(),
}).then(decode),
).resolves.toBe("# icon.svg\n\nIcon\n\nHello wiki\n");
for (const [filename, mediaType] of [
["photo.png", PNG_MEDIA_TYPE],
["photo.heic", "image/heic"],
["photo.jpg", "application/octet-stream"],
] as const) {
await expect(
normalizeDocumentContent({ filename, mediaType, source: pngFixture() }),
).rejects.toMatchObject({ code: "DOCUMENT_SOURCE_UNSUPPORTED" });
}
});
it("rejects nested gzip instead of recursing through wrappers", async () => {
let source = encode("# From nested gzip\n");
let filename = "notes.md";
for (let layer = 0; layer < 8; layer += 1) {
source = gzipFixture(source);
filename += ".gz";
}
await expect(
normalizeDocumentContent({
filename,
mediaType: GZIP_MEDIA_TYPE,
source,
}),
).rejects.toMatchObject({
code: "DOCUMENT_SOURCE_UNSUPPORTED",
disposition: "terminal",
});
});
it("does not keep unwrapping gzip members inside a ZIP", async () => {
await expect(
normalizeDocumentContent({
filename: "bundle.zip",
mediaType: ZIP_MEDIA_TYPE,
source: zipFixture([
{ name: "notes.md", value: "# From zip\n" },
{
name: "nested.md.gz.gz",
value: gzipFixture(gzipFixture(encode("# From nested gzip\n"))),
},
]),
}).then(decode),
).resolves.toBe("## notes.md\n\n# From zip\n");
});
it("rejects a gzip-wrapped ZIP as a second archive layer", async () => {
await expect(
normalizeDocumentContent({
filename: "bundle.zip.gz",
mediaType: GZIP_MEDIA_TYPE,
source: gzipFixture(
zipFixture([{ name: "notes.md", value: "# From zip\n" }]),
),
}),
).rejects.toMatchObject({
code: "DOCUMENT_SOURCE_UNSUPPORTED",
disposition: "terminal",
});
});
it("treats an Office Open XML file labeled as ZIP as that Office kind", async () => {
await expect(
normalizeDocumentContent({
filename: "budget.xlsx",
mediaType: ZIP_MEDIA_TYPE,
representationFormat: "binary",
source: xlsxFixture(),
}).then(decode),
).resolves.toBe(
"## People\n\n| Name | Count |\n| --- | --- |\n| Ada | 42 |\n",
);
});
it("keeps SVG in-process and sends ZIP to the worker path", () => {
expect(
isInProcessDocumentSource({
filename: "icon.svg",
mediaType: SVG_MEDIA_TYPE,
}),
).toBe(true);
expect(
isInProcessDocumentSource({
filename: "bundle.zip",
mediaType: ZIP_MEDIA_TYPE,
}),
).toBe(false);
});
it("keeps UTF-8 Markdown and treats a generic .md Version as text", async () => {
await expect(
normalizeDocumentContent({
mediaType: "text/markdown; charset=utf-8",
source: encode("# Notes\r\n"),
}).then(decode),
).resolves.toBe("# Notes\n");
await expect(
normalizeDocumentContent({
filename: "notes.md",
mediaType: "application/octet-stream",
source: encode("# Notes\n"),
}).then(decode),
).resolves.toBe("# Notes\n");
});
it.each([
["text/csv", ","],
["text/tab-separated-values", "\t"],
])(
"normalizes large delimited documents within the byte limits: %s",
async (mediaType, delimiter) => {
const rows = 200_000;
const source = encode(
`name${delimiter}count\n${`item${delimiter}1\n`.repeat(rows)}`,
);
const expected = `| name | count |\n| --- | --- |\n${"| item | 1 |\n".repeat(rows)}`;
expect(source.byteLength).toBeLessThan(MAX_DOCUMENT_SOURCE_BYTES);
expect(encode(expected).byteLength).toBeLessThan(
MAX_DOCUMENT_MARKDOWN_BYTES,
);
await expect(
normalizeDocumentContent({ mediaType, source }).then(decode),
).resolves.toBe(expected);
},
);
it("keeps the Markdown size error for a large delimited document", async () => {
await expect(
normalizeDocumentContent(
{
mediaType: "text/csv",
source: encode(`name,count\n${"item,1\n".repeat(200_000)}`),
},
1024,
),
).rejects.toMatchObject({ code: "DOCUMENT_MARKDOWN_TOO_LARGE" });
});
it("normalizes LF and one trailing newline and enforces output size", () => {
expect(decode(canonicalizeMarkdown("Cafe\u0301\r\nline\r\n\r\n"))).toBe(
"Caf\u00e9\nline\n",
);
expect(() => canonicalizeMarkdown("1234", 4)).toThrowError(
expect.objectContaining({ code: "DOCUMENT_MARKDOWN_TOO_LARGE" }),
);
});
it.each([
{
code: "DOCUMENT_SOURCE_UNSUPPORTED",
mediaType: "audio/mpeg",
source: encode("not audio"),
},
{
code: "DOCUMENT_SOURCE_UNSUPPORTED",
mediaType: "application/vnd.ms-excel",
source: encode("not a spreadsheet"),
},
{
code: "DOCUMENT_SOURCE_INVALID",
mediaType: PDF_MEDIA_TYPE,
source: encode("not a PDF"),
},
{
code: "DOCUMENT_SOURCE_INVALID",
mediaType: DOCX_MEDIA_TYPE,
source: encode("not a DOCX"),
},
{
code: "DOCUMENT_SOURCE_INVALID",
mediaType: DOC_MEDIA_TYPE,
source: encode("not a DOC"),
},
{
code: "DOCUMENT_SOURCE_EMPTY",
mediaType: PDF_MEDIA_TYPE,
source: pdfFixture(""),
},
])("returns terminal $code for unusable input", async (fixture) => {
await expect(normalizeDocumentContent(fixture)).rejects.toMatchObject({
code: fixture.code,
disposition: "terminal",
});
});
it("converts one document at a time and gives each conversion its own deadline", async () => {
let running = 0;
let peak = 0;
const normalizer = new WorkerDocumentNormalizer({
execute: async () => {
running += 1;
peak = Math.max(peak, running);
await new Promise((resolve) => setTimeout(resolve, 60));
running -= 1;
return new TextEncoder().encode("ok\n");
},
timeoutMs: 100,
});
// The second request finishes after 120 ms, past one timeout.
await expect(
Promise.all([1, 2].map(() => normalizer.normalizeDocument(pdf()))),
).resolves.toHaveLength(2);
expect(peak).toBe(1);
});
it("converts text in process without waiting for the worker", async () => {
const normalizer = new WorkerDocumentNormalizer({
execute: async () => await new Promise<Uint8Array>(() => undefined),
});
void normalizer.normalizeDocument(pdf()).catch(() => undefined);
await expect(
normalizer
.normalizeDocument({
filename: "notes.md",
mediaType: "text/markdown",
source: encode("# Notes"),
})
.then(decode),
).resolves.toBe("# Notes\n");
});
it("classifies a conversion that runs past its deadline as terminal", async () => {
const normalizer = new WorkerDocumentNormalizer({
execute: async () => await new Promise<Uint8Array>(() => undefined),
timeoutMs: 5,
});
await expect(normalizer.normalizeDocument(pdf())).rejects.toMatchObject({
code: "DOCUMENT_NORMALIZATION_TIMEOUT",
disposition: "terminal",
status: 422,
});
});
it("rejects a request when the queue is full", async () => {
let finish!: () => void;
const normalizer = new WorkerDocumentNormalizer({
execute: async () => {
await new Promise<void>((resolve) => {
finish = resolve;
});
return encode("ok\n");
},
maxQueued: 1,
});
const converting = normalizer.normalizeDocument(pdf());
const waiting = normalizer.normalizeDocument(pdf());
await expect(normalizer.normalizeDocument(pdf())).rejects.toMatchObject({
code: "DOCUMENT_NORMALIZATION_UNAVAILABLE",
disposition: "retryable",
});
finish();
await converting;
finish();
await expect(waiting.then(decode)).resolves.toBe("ok\n");
});
it("drops a waiting request when its caller goes away", async () => {
const started: string[] = [];
let finish!: () => void;
const normalizer = new WorkerDocumentNormalizer({
execute: async (input) => {
started.push(input.filename ?? "");
if (input.filename === "first.pdf") {
await new Promise<void>((resolve) => {
finish = resolve;
});
}
return encode("ok\n");
},
});
const caller = new AbortController();
const first = normalizer.normalizeDocument(pdf("first.pdf"));
const dropped = normalizer.normalizeDocument(
pdf("dropped.pdf"),
caller.signal,
);
const third = normalizer.normalizeDocument(pdf("third.pdf"));
caller.abort();
await expect(dropped).rejects.toMatchObject({
code: "DOCUMENT_NORMALIZATION_UNAVAILABLE",
});
finish();
await Promise.all([first, third]);
expect(started).toEqual(["first.pdf", "third.pdf"]);
});
it("rejects a source past the byte budget until admitted sources settle", async () => {
let finish!: () => void;
const normalizer = new WorkerDocumentNormalizer({
execute: async (input) => {
if (input.filename === "held.pdf") {
await new Promise<void>((resolve) => {
finish = resolve;
});
}
return encode("ok\n");
},
maxAdmittedBytes: MAX_DOCUMENT_SOURCE_BYTES,
});
const held = normalizer.normalizeDocument(sized(1, "held.pdf"));
await expect(
normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES)),
).rejects.toMatchObject({
code: "DOCUMENT_NORMALIZATION_UNAVAILABLE",
disposition: "retryable",
});
await waitFor(() => finish !== undefined);
finish();
await held;
// Alone, a maximum source fits even though the budget has no headroom.
await expect(
normalizer
.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES))
.then(decode),
).resolves.toBe("ok\n");
});
it("counts in-process text sources against the byte budget", async () => {
const normalizer = new WorkerDocumentNormalizer({
execute: async () => encode("ok\n"),
maxAdmittedBytes: MAX_DOCUMENT_SOURCE_BYTES,
});
// Admission is synchronous, so the text source is admitted before the
// worker source is called.
const text = normalizer.normalizeDocument({
filename: "notes.md",
mediaType: "text/markdown",
source: encode("# Notes"),
});
await expect(
normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES)),
).rejects.toMatchObject({ code: "DOCUMENT_NORMALIZATION_UNAVAILABLE" });
await expect(text.then(decode)).resolves.toBe("# Notes\n");
await expect(
normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES)),
).resolves.toBeDefined();
});
it("releases the byte budget after a conversion failure", async () => {
const normalizer = new WorkerDocumentNormalizer({
execute: async (input) => {
if (input.filename === "broken.pdf") {
throw new DocumentNormalizationError("DOCUMENT_SOURCE_INVALID");
}
return encode("ok\n");
},
maxAdmittedBytes: MAX_DOCUMENT_SOURCE_BYTES,
});
await expect(
normalizer.normalizeDocument(sized(1, "broken.pdf")),
).rejects.toMatchObject({ code: "DOCUMENT_SOURCE_INVALID" });
const first = normalizer.normalizeDocument(sized(1));
await expect(
normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES - 1)),
).resolves.toBeDefined();
await first;
});
it("releases the byte budget of a waiting request whose caller goes away", async () => {
let finish!: () => void;
const normalizer = new WorkerDocumentNormalizer({
execute: async (input) => {
if (input.filename === "held.pdf") {
await new Promise<void>((resolve) => {
finish = resolve;
});
}
return encode("ok\n");
},
maxAdmittedBytes: 2 * MAX_DOCUMENT_SOURCE_BYTES,
});
const caller = new AbortController();
const held = normalizer.normalizeDocument(sized(1, "held.pdf"));
const dropped = normalizer.normalizeDocument(
sized(MAX_DOCUMENT_SOURCE_BYTES),
caller.signal,
);
caller.abort();
await expect(dropped).rejects.toMatchObject({
code: "DOCUMENT_NORMALIZATION_UNAVAILABLE",
});
// A leaked reservation would put this request past the budget.
const next = normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES));
await waitFor(() => finish !== undefined);
finish();
await Promise.all([held, next]);
});
it("releases the byte budget of a request whose wait times out", async () => {
const normalizer = new WorkerDocumentNormalizer({
execute: async (input) => {
if (input.filename === "next.pdf") return encode("ok\n");
return await new Promise<Uint8Array>(() => undefined);
},
maxAdmittedBytes: 2 * MAX_DOCUMENT_SOURCE_BYTES,
timeoutMs: 100,
});
// The first two conversions each run out their 100 ms deadline, so the
// worker is busy from 0 to 200 ms; the second request, queued at 10 ms,
// gets the worker at 100 ms, and the one queued at 60 ms gives up waiting
// at 160 ms.
vi.useFakeTimers();
try {
const ignore = (): undefined => undefined;
const first = normalizer.normalizeDocument(sized(1)).catch(ignore);
await vi.advanceTimersByTimeAsync(10);
const second = normalizer.normalizeDocument(sized(1)).catch(ignore);
await vi.advanceTimersByTimeAsync(50);
const waiting = expect(
normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES)),
).rejects.toMatchObject({ code: "DOCUMENT_NORMALIZATION_UNAVAILABLE" });
await vi.advanceTimersByTimeAsync(100);
await waiting;
// A leaked reservation would put this request past the budget.
const next = normalizer.normalizeDocument(
sized(MAX_DOCUMENT_SOURCE_BYTES, "next.pdf"),
);
await vi.advanceTimersByTimeAsync(50);
await Promise.all([first, second]);
await expect(next.then(decode)).resolves.toBe("ok\n");
} finally {
vi.useRealTimers();
}
});
it("releases the byte budget of a request rejected by a full queue", async () => {
const finishes: (() => void)[] = [];
const normalizer = new WorkerDocumentNormalizer({
execute: async () => {
await new Promise<void>((resolve) => {
finishes.push(resolve);
});
return encode("ok\n");
},
maxAdmittedBytes: 2 * MAX_DOCUMENT_SOURCE_BYTES + 1,
maxQueued: 1,
});
const converting = normalizer.normalizeDocument(sized(1));
const waiting = normalizer.normalizeDocument(
sized(MAX_DOCUMENT_SOURCE_BYTES),
);
await expect(
normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES)),
).rejects.toMatchObject({ code: "DOCUMENT_NORMALIZATION_UNAVAILABLE" });
await waitFor(() => finishes.length === 1);
finishes[0]!();
await converting;
// The waiting source now converts; a leaked reservation would put this
// request past the budget.
const next = normalizer.normalizeDocument(sized(MAX_DOCUMENT_SOURCE_BYTES));
await waitFor(() => finishes.length === 2);
finishes[1]!();
await waiting;
await waitFor(() => finishes.length === 3);
finishes[2]!();
await expect(next.then(decode)).resolves.toBe("ok\n");
});
it("requires a byte budget that admits one maximum source", () => {
expect(
() =>
new WorkerDocumentNormalizer({
maxAdmittedBytes: MAX_DOCUMENT_SOURCE_BYTES - 1,
}),
).toThrow("byte budget");
});
});
function pdf(filename = "notes.pdf") {
return { filename, mediaType: PDF_MEDIA_TYPE, source: pdfFixture() };
}
const maxSizedSource = new Uint8Array(MAX_DOCUMENT_SOURCE_BYTES);
function sized(byteLength: number, filename = "sized.pdf") {
return {
filename,
mediaType: PDF_MEDIA_TYPE,
source: maxSizedSource.subarray(0, byteLength),
};
}
async function waitFor(condition: () => boolean): Promise<void> {
while (!condition()) {
await new Promise((resolve) => setTimeout(resolve, 1));
}
}