Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
import { describe, expect, it } from "vitest";
import {
concat2,
decodeBase64,
decodeUtf8,
encodeBase64,
encodeUtf8,
processStringOrBuffer,
stripBom,
truncateUtf8Bytes,
unwrapStringOrBuffer,
wrapStringOrBuffer
} from "./binary.js";
describe("binary utils", () => {
describe("concat2", () => {
it("joins two byte arrays preserving order and length", () => {
const a = new Uint8Array([1, 2, 3]);
const b = new Uint8Array([4, 5]);
const out = concat2(a, b);
expect(out).toBeInstanceOf(Uint8Array);
expect(out.length).toBe(5);
expect(Array.from(out)).toEqual([1, 2, 3, 4, 5]);
});
it("handles empty inputs on either side", () => {
const empty = new Uint8Array([]);
const data = new Uint8Array([9, 8]);
expect(Array.from(concat2(empty, data))).toEqual([9, 8]);
expect(Array.from(concat2(data, empty))).toEqual([9, 8]);
expect(concat2(empty, empty).length).toBe(0);
});
it("does not mutate the source arrays", () => {
const a = new Uint8Array([1, 2]);
const b = new Uint8Array([3, 4]);
concat2(a, b);
expect(Array.from(a)).toEqual([1, 2]);
expect(Array.from(b)).toEqual([3, 4]);
});
});
describe("encodeUtf8 / decodeUtf8", () => {
it("encodes a string into its UTF-8 byte representation", () => {
const bytes = encodeUtf8("AB");
expect(bytes).toBeInstanceOf(Uint8Array);
expect(Array.from(bytes)).toEqual([0x41, 0x42]);
});
it("encodes multi-byte (non-ASCII) characters correctly", () => {
// "é" => 0xC3 0xA9, "€" => 0xE2 0x82 0xAC
expect(Array.from(encodeUtf8("é"))).toEqual([0xc3, 0xa9]);
expect(Array.from(encodeUtf8("€"))).toEqual([0xe2, 0x82, 0xac]);
});
it("accepts a buffer and re-encodes its decoded form", () => {
const buffer = new Uint8Array([0xc3, 0xa9]); // "é"
expect(Array.from(encodeUtf8(buffer))).toEqual([0xc3, 0xa9]);
});
it("decodes a UTF-8 buffer back into a string", () => {
expect(decodeUtf8(new Uint8Array([0x41, 0x42]))).toBe("AB");
expect(decodeUtf8(new Uint8Array([0xc3, 0xa9]))).toBe("é");
});
it("returns the string unchanged when given a string", () => {
expect(decodeUtf8("already a string")).toBe("already a string");
});
it("round-trips arbitrary unicode through encode then decode", () => {
const original = "Hello, 世界! €é";
expect(decodeUtf8(encodeUtf8(original))).toBe(original);
});
});
describe("wrapStringOrBuffer", () => {
it("converts a string to a UTF-8 Uint8Array", () => {
const out = wrapStringOrBuffer("AB");
expect(out).toBeInstanceOf(Uint8Array);
expect(Array.from(out)).toEqual([0x41, 0x42]);
});
it("returns a buffer unchanged (same reference)", () => {
const buffer = new Uint8Array([1, 2, 3]);
expect(wrapStringOrBuffer(buffer)).toBe(buffer);
});
});
describe("unwrapStringOrBuffer", () => {
it("returns a string unchanged", () => {
expect(unwrapStringOrBuffer("hello")).toBe("hello");
});
it("decodes a buffer to its UTF-8 string", () => {
expect(unwrapStringOrBuffer(new Uint8Array([0xc3, 0xa9]))).toBe("é");
});
});
describe("encodeBase64 / decodeBase64", () => {
it("encodes a string to base64", () => {
// "Man" => "TWFu" is the canonical RFC 4648 example
expect(encodeBase64("Man")).toBe("TWFu");
});
it("encodes a byte buffer to base64", () => {
expect(encodeBase64(new Uint8Array([0x4d, 0x61, 0x6e]))).toBe("TWFu");
});
it("produces padding for inputs whose length is not a multiple of 3", () => {
expect(encodeBase64("M")).toBe("TQ==");
expect(encodeBase64("Ma")).toBe("TWE=");
});
it("decodes base64 back into the original bytes", () => {
expect(Array.from(decodeBase64("TWFu"))).toEqual([0x4d, 0x61, 0x6e]);
});
it("round-trips multi-byte unicode content", () => {
const original = "café — 世界";
const encoded = encodeBase64(original);
const decoded = decodeUtf8(decodeBase64(encoded));
expect(decoded).toBe(original);
});
it("handles empty input", () => {
expect(encodeBase64("")).toBe("");
expect(decodeBase64("").length).toBe(0);
});
it("preserves raw binary bytes through a buffer round-trip", () => {
const bytes = new Uint8Array([0, 255, 128, 1, 254]);
expect(Array.from(decodeBase64(encodeBase64(bytes)))).toEqual([0, 255, 128, 1, 254]);
});
it("matches the RFC 4648 §10 vectors", () => {
expect(encodeBase64("")).toBe("");
expect(encodeBase64("f")).toBe("Zg==");
expect(encodeBase64("fo")).toBe("Zm8=");
expect(encodeBase64("foob")).toBe("Zm9vYg==");
expect(encodeBase64("fooba")).toBe("Zm9vYmE=");
expect(encodeBase64("foobar")).toBe("Zm9vYmFy");
});
it("round-trips every byte value 0..255 through the active provider", () => {
const bytes = new Uint8Array(256);
for (let i = 0; i < 256; i++) bytes[i] = i;
expect(Array.from(decodeBase64(encodeBase64(bytes)))).toEqual(Array.from(bytes));
});
it("round-trips a large buffer", () => {
const bytes = new Uint8Array(100_000);
for (let i = 0; i < bytes.length; i++) bytes[i] = (i * 31 + 7) & 0xff;
expect(Array.from(decodeBase64(encodeBase64(bytes)))).toEqual(Array.from(bytes));
});
});
describe("stripBom", () => {
const BOM = String.fromCharCode(0xfeff);
it("strips a single leading BOM (U+FEFF), leaving the remainder intact", () => {
expect(stripBom(BOM + "hello")).toBe("hello");
// only the first code point is removed, so a second BOM survives
expect(stripBom(BOM + BOM + "x")).toBe(BOM + "x");
});
it("returns the string unchanged when there is no leading BOM", () => {
expect(stripBom("hello")).toBe("hello");
expect(stripBom("")).toBe("");
// a BOM that is not at the start is preserved
expect(stripBom("a" + BOM + "b")).toBe("a" + BOM + "b");
});
});
describe("truncateUtf8Bytes", () => {
it("returns the string unchanged when it already fits", () => {
expect(truncateUtf8Bytes("hello", 255)).toBe("hello");
expect(truncateUtf8Bytes("hello", 5)).toBe("hello");
expect(truncateUtf8Bytes("", 0)).toBe("");
});
it("truncates ASCII to an exact byte budget", () => {
expect(truncateUtf8Bytes("hello world", 5)).toBe("hello");
expect(encodeUtf8(truncateUtf8Bytes("hello world", 5)).length).toBe(5);
});
it("never splits a 2-byte character and stays within the budget", () => {
// "é" is 2 bytes; a budget of 1 cannot hold it, so it is dropped entirely.
expect(truncateUtf8Bytes("aé", 2)).toBe("a");
// a budget of 3 fits "aé" (1 + 2 bytes) exactly.
expect(truncateUtf8Bytes("aé", 3)).toBe("aé");
// a budget of 2 in the middle of "é" must back up to the boundary.
const truncated = truncateUtf8Bytes("aéb", 2);
expect(truncated).toBe("a");
expect(encodeUtf8(truncated).length).toBeLessThanOrEqual(2);
});
it("never splits a 3-byte or 4-byte character (CJK / emoji)", () => {
// "世" is 3 bytes; budgets below 3 drop it, exactly 3 keeps it.
expect(truncateUtf8Bytes("世", 2)).toBe("");
expect(truncateUtf8Bytes("世", 3)).toBe("世");
// "😀" is 4 bytes (a surrogate pair in JS); a 3-byte budget drops it whole.
expect(truncateUtf8Bytes("😀", 3)).toBe("");
expect(truncateUtf8Bytes("😀", 4)).toBe("😀");
});
it("keeps the result valid (no replacement characters) when cutting multi-byte text", () => {
const text = "汉".repeat(10); // 30 bytes
const result = truncateUtf8Bytes(text, 16); // 16 / 3 = 5 full chars
expect(result).toBe("汉".repeat(5));
expect(result).not.toContain("�");
expect(encodeUtf8(result).length).toBeLessThanOrEqual(16);
});
});
describe("processStringOrBuffer", () => {
it("returns an empty string for nullish or empty input", () => {
expect(processStringOrBuffer(null)).toBe("");
expect(processStringOrBuffer("")).toBe("");
expect(processStringOrBuffer(new Uint8Array([]))).toBe("");
});
it("returns a string unchanged without transformation", () => {
expect(processStringOrBuffer("plain text")).toBe("plain text");
});
it("decodes a plain UTF-8 buffer", () => {
const buffer = encodeUtf8("Hello UTF-8 world with enough text to detect");
expect(processStringOrBuffer(buffer)).toBe("Hello UTF-8 world with enough text to detect");
});
it("decodes multi-byte (CJK) UTF-8 content", () => {
const buffer = encodeUtf8("中文测试内容");
expect(processStringOrBuffer(buffer)).toBe("中文测试内容");
});
it("decodes a UTF-16LE buffer with a BOM and strips the BOM", () => {
// BOM (0xFF 0xFE) followed by "Hi" in little-endian UTF-16
const buffer = new Uint8Array([0xff, 0xfe, 0x48, 0x00, 0x69, 0x00]);
const result = processStringOrBuffer(buffer);
expect(result).toBe("Hi");
// ensure the BOM character (U+FEFF) was stripped
expect(result.charCodeAt(0)).not.toBe(0xfeff);
});
it("treats UTF-16 without a BOM as UTF-8 (matching the previous chardet behaviour)", () => {
// "Hi" in little-endian UTF-16 but with no BOM: chardet never classified this as UTF-16LE
// either, so it was — and still is — decoded byte-wise as UTF-8 (null high bytes preserved).
const buffer = new Uint8Array([0x48, 0x00, 0x69, 0x00]);
expect([...processStringOrBuffer(buffer)].map((c) => c.charCodeAt(0))).toEqual([0x48, 0x00, 0x69, 0x00]);
});
it("decodes a plain ASCII buffer as UTF-8", () => {
const buffer = new Uint8Array([0x61, 0x62, 0x63]); // "abc"
expect(processStringOrBuffer(buffer)).toBe("abc");
});
});
});