Skip to content

Build your personal knowledge base with Trilium Notes

15.6 KB
Raw Blame
/**
 * Tests the electron-forge package of the desktop app from outside the repository, where Node's
 * module resolution can no longer walk up into the workspace's `node_modules` and hide a package
 * the build forgot to ship.
 *
 * 1. Moves `dist/out/<product>-<platform>-<arch>` into the OS temp directory.
 * 2. Checks that every package a bundle leaves external is inside `app.asar`, together with the
 *    dependencies it needs, read from the per-entry metafiles under `meta/` in the archive.
 * 3. Launches the executable there, with throwaway data directories and an environment stripped
 *    of anything pointing into the repository, and waits for its HTTP server to answer.
 *    `--smoke` then quits it; otherwise the app stays open for manual testing.
 *
 * On NixOS the stock Electron binary runs through nix-ld, which this script points at the shared
 * libraries Electron links against, built from the nixpkgs revision in `flake.lock`.
 *
 * `electron-forge:test-packaged` packages first and then runs this script.
 */

import { spawn, spawnSync } from "child_process";
import {
    closeSync, cpSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync,
    renameSync, rmSync
} from "fs";
import { builtinModules } from "module";
import { tmpdir } from "os";
import { delimiter, isAbsolute, join, relative } from "path";

import { PRODUCT_NAME } from "../src/app-info.js";

const DESKTOP_DIR = join(__dirname, "..");
const REPO_DIR = join(DESKTOP_DIR, "../..");
const TEST_DIR = join(tmpdir(), "trilium-packaged-test");
/** Matches `executableName` in `electron-forge/forge.config.ts`. */
const EXECUTABLE_NAME = "trilium";
const PORT = 37850;
const STARTUP_TIMEOUT_MS = 120_000;
/** The nixpkgs attributes holding the shared libraries Electron links against. */
const NIXOS_ELECTRON_LIBRARIES = [
    "alsa-lib", "at-spi2-atk", "at-spi2-core", "cairo", "cups", "dbus", "expat", "glib", "gtk3",
    "libdrm", "libGL", "libgbm", "libxkbcommon", "mesa", "nspr", "nss", "pango", "systemd",
    "xorg.libX11", "xorg.libXcomposite", "xorg.libXdamage", "xorg.libXext", "xorg.libXfixes",
    "xorg.libXrandr", "xorg.libxcb", "xorg.libxshmfence"
];

/** Workspace packages that a packaged app never loads, so they are not shipped. */
const NOT_SHIPPED: Record<string, string> = {
    "@agentclientprotocol/codex-acp": "copied to assets/codex-acp.mjs; resolved only from source",
    "@triliumnext/core": "test fixtures and schema.sql, reached only in tests or from source",
    "node-addon-api": "C++ headers for compiling better-sqlite3; the shipped prebuild needs none",
    "node-fetch": "tesseract.js requires it only when global fetch is missing",
    "pdfjs-dist": "officeparser loads it for PDFs, which go through unpdf instead"
};

async function main() {
    const smoke = process.argv.includes("--smoke");
    const appDir = moveOutOfRepo();
    console.log(`Testing the package at ${appDir}`);

    const missing = findMissingPackages(new Asar(join(getResourcesDir(appDir), "app.asar")));
    if (missing.length) {
        console.error("\nPackages the app loads but the package does not contain:");
        for (const { name, reason } of missing) {
            console.error(`  ${name} (${reason})`);
        }
        console.error("");
        process.exitCode = 1;
    } else {
        console.log("Every external package ships inside app.asar.");
    }

    await launch(appDir, smoke);
}

function moveOutOfRepo() {
    const packageName = `${PRODUCT_NAME}-${process.platform}-${process.arch}`;
    // forge resolves its `outDir` against the packaged directory, `dist`.
    const source = join(DESKTOP_DIR, "dist", "out", packageName);
    if (!existsSync(source)) {
        throw new Error(`No package at ${source}. Run `
            + `"pnpm --filter @triliumnext/desktop electron-forge:package" first.`);
    }
    if (isInside(REPO_DIR, TEST_DIR)) {
        throw new Error(`The temp directory ${TEST_DIR} is inside the repository.`);
    }

    rmSync(TEST_DIR, { recursive: true, force: true });
    mkdirSync(TEST_DIR, { recursive: true });
    const destination = join(TEST_DIR, packageName);
    try {
        renameSync(source, destination);
    } catch (e) {
        if ((e as NodeJS.ErrnoException).code !== "EXDEV") {
            throw e;
        }
        cpSync(source, destination, { recursive: true, verbatimSymlinks: true });
        rmSync(source, { recursive: true, force: true });
    }
    return destination;
}

function getResourcesDir(appDir: string) {
    return process.platform === "darwin"
        ? join(appDir, `${PRODUCT_NAME}.app`, "Contents", "Resources")
        : join(appDir, "resources");
}

function getExecutablePath(appDir: string) {
    switch (process.platform) {
        case "darwin":
            return join(appDir, `${PRODUCT_NAME}.app`, "Contents", "MacOS", PRODUCT_NAME);
        case "win32": return join(appDir, `${EXECUTABLE_NAME}.exe`);
        default: return join(appDir, EXECUTABLE_NAME);
    }
}

/**
 * Lists the packages imported by the bundles, or required by a shipped package, that the archive
 * does not contain but the workspace does. A package the workspace lacks as well fails the same
 * way in development, which makes it an optional import (`ws` probing for `bufferutil`), not a
 * hidden dependency. The same test selects the optional dependencies that apply to this machine,
 * since pnpm installs only the platform-specific binary packages matching it.
 */
function findMissingPackages(asar: Asar) {
    const missing: { name: string; reason: string }[] = [];
    const visited = new Set<string>();

    const visit = (name: string, fromDir: string, reason: string) => {
        if (name in NOT_SHIPPED || !isInstalledInWorkspace(name)) {
            return;
        }
        const dir = [ `${fromDir}/node_modules/${name}`, `node_modules/${name}` ]
            .find((d) => asar.has(`${d}/package.json`));
        if (!dir) {
            missing.push({ name, reason });
            return;
        }
        if (visited.has(dir)) {
            return;
        }
        visited.add(dir);

        const pkg = JSON.parse(asar.read(`${dir}/package.json`));
        for (const dep of Object.keys(pkg.dependencies ?? {})) {
            visit(dep, dir, `dependency of ${name}`);
        }
        for (const dep of Object.keys(pkg.optionalDependencies ?? {})) {
            visit(dep, dir, `optional dependency of ${name}, installed for this platform`);
        }
    };

    for (const [ name, importers ] of getExternalPackages(asar)) {
        visit(name, "", `imported by ${[ ...importers ].join(", ")}`);
    }
    return missing;
}

/** Maps each external package in the bundles' metafiles to the output files importing it. */
function getExternalPackages(asar: Asar) {
    const metaFiles = asar.list("meta").filter((f) => f.endsWith(".json"));
    if (!metaFiles.length) {
        throw new Error("The package has no meta/*.json metafiles; package it again.");
    }

    const packages = new Map<string, Set<string>>();
    for (const metaFile of metaFiles) {
        const { outputs } = JSON.parse(asar.read(`meta/${metaFile}`)) as {
            outputs: Record<string, { imports: { path: string; external?: boolean }[] }>;
        };
        for (const [ outputPath, output ] of Object.entries(outputs)) {
            for (const { path, external } of output.imports) {
                const name = external && getPackageName(path);
                if (name) {
                    const importers = packages.get(name) ?? new Set<string>();
                    importers.add(outputPath.replace(/^dist\//, ""));
                    packages.set(name, importers);
                }
            }
        }
    }
    return packages;
}

/** Returns the package an import specifier names, or `null` for a relative path or a built-in. */
function getPackageName(specifier: string) {
    if (specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("node:")) {
        return null;
    }
    const segments = specifier.split("/");
    const name = specifier.startsWith("@") ? segments.slice(0, 2).join("/") : segments[0];
    const isBuiltin = builtinModules.includes(name) || builtinModules.includes(specifier);
    if (name === "electron" || isBuiltin) {
        return null;
    }
    return name;
}

function isInstalledInWorkspace(name: string) {
    return [ REPO_DIR, DESKTOP_DIR ].some((dir) => existsSync(join(dir, "node_modules", name)));
}

async function launch(appDir: string, smoke: boolean) {
    const url = `http://localhost:${PORT}/`;
    if (await responds(url)) {
        throw new Error(`Something already answers on ${url}; close it first.`);
    }

    const dataDir = join(TEST_DIR, "data");
    const cleanEnv = getCleanEnv();
    const env = {
        ...cleanEnv,
        ...getNixOsEnv(cleanEnv),
        TRILIUM_DATA_DIR: dataDir,
        TRILIUM_ELECTRON_DATA_DIR: join(TEST_DIR, "data-electron"),
        TRILIUM_PORT: String(PORT)
    };
    const startedAt = Date.now();
    const child = spawn(getExecutablePath(appDir), [], { cwd: TEST_DIR, env, stdio: "inherit" });
    let exitCode: number | null | undefined;
    const exited = new Promise<void>((resolve) => child.on("exit", (code) => {
        exitCode = code;
        resolve();
    }));

    while (exitCode === undefined && !(await responds(url))) {
        if (Date.now() - startedAt > STARTUP_TIMEOUT_MS) {
            console.error(`No answer on ${url} within ${STARTUP_TIMEOUT_MS / 1000} s.`);
            printLogTail(dataDir);
            killTree(child.pid);
            process.exit(1);
        }
        await new Promise((resolve) => setTimeout(resolve, 500));
    }

    if (exitCode !== undefined) {
        console.error(`The app exited with code ${exitCode} before its server answered.`);
        printLogTail(dataDir);
        process.exit(1);
    }

    const seconds = ((Date.now() - startedAt) / 1000).toFixed(1);
    console.log(`The server answered on ${url} after ${seconds} s.`);
    if (smoke) {
        killTree(child.pid);
    } else {
        console.log(`Data directory: ${dataDir}\nClose the app to finish.`);
    }
    await exited;
}

/**
 * Copies the environment without package-manager, Node, Electron and Trilium variables, or any
 * value pointing into the repository.
 */
function getCleanEnv() {
    const env: Record<string, string> = {};
    for (const [ key, value ] of Object.entries(process.env)) {
        if (value === undefined || /^(npm_|pnpm_|node_|electron_|trilium_)|^INIT_CWD$/i.test(key)) {
            continue;
        }
        if (key.toUpperCase() === "PATH") {
            const entries = value.split(delimiter).filter((entry) => !isInside(REPO_DIR, entry));
            env[key] = entries.join(delimiter);
        } else if (!value.toLowerCase().includes(REPO_DIR.toLowerCase())) {
            env[key] = value;
        }
    }
    return env;
}

/**
 * On NixOS, returns a `NIX_LD_LIBRARY_PATH` that adds Electron's shared libraries to those of
 * `programs.nix-ld.libraries`. Elsewhere, returns no variables.
 */
function getNixOsEnv(env: Record<string, string>): Record<string, string> {
    if (process.platform !== "linux" || !existsSync("/etc/NIXOS")) {
        return {};
    }
    if (!env.NIX_LD) {
        throw new Error("NixOS runs the packaged app only through nix-ld; "
            + "enable it with `programs.nix-ld.enable = true;`.");
    }

    const lock = JSON.parse(readFileSync(join(REPO_DIR, "flake.lock"), "utf-8"));
    const { owner, repo, rev, narHash } = lock.nodes[lock.nodes.root.inputs.nixpkgs].locked;
    const nixpkgs = `github:${owner}/${repo}/${rev}?narHash=${encodeURIComponent(narHash)}`;
    const expression = `
        let pkgs = (builtins.getFlake "${nixpkgs}").legacyPackages.\${builtins.currentSystem};
        in pkgs.symlinkJoin {
            name = "trilium-electron-libs";
            paths = map pkgs.lib.getLib (with pkgs; [ ${NIXOS_ELECTRON_LIBRARIES.join(" ")} ]);
        }`;
    console.log("Building Electron's shared libraries for nix-ld.");
    const build = spawnSync(
        "nix", [ "build", "--impure", "--no-link", "--print-out-paths", "--expr", expression ],
        { encoding: "utf-8", stdio: [ "ignore", "pipe", "inherit" ] }
    );
    if (build.status !== 0) {
        const reason = build.error ?? `exit code ${build.status}`;
        throw new Error(`Building Electron's shared libraries failed (${reason}).`);
    }

    const libDir = join(build.stdout.trim(), "lib");
    return {
        NIX_LD_LIBRARY_PATH: [ libDir, env.NIX_LD_LIBRARY_PATH ].filter(Boolean).join(delimiter)
    };
}

async function responds(url: string) {
    try {
        await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(2000) });
        return true;
    } catch {
        return false;
    }
}

function printLogTail(dataDir: string) {
    const logDir = join(dataDir, "log");
    const logFile = existsSync(logDir) ? readdirSync(logDir).sort().at(-1) : undefined;
    if (!logFile) {
        console.error(`No log file under ${logDir}.`);
        return;
    }
    const lines = readFileSync(join(logDir, logFile), "utf-8").trimEnd().split("\n");
    console.error(`\nLast lines of ${join(logDir, logFile)}:\n${lines.slice(-40).join("\n")}`);
}

function killTree(pid: number | undefined) {
    if (pid === undefined) {
        return;
    }
    if (process.platform === "win32") {
        spawnSync("taskkill", [ "/pid", String(pid), "/T", "/F" ], { stdio: "ignore" });
    } else {
        process.kill(pid);
    }
}

function isInside(parent: string, child: string) {
    const rel = relative(parent, child);
    return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
}

/**
 * Reads an asar archive: a pickled JSON header listing every file, followed by the packed file
 * contents. Files marked `unpacked` live in `app.asar.unpacked` beside it.
 */
class Asar {
    private readonly header: AsarDirectory;
    private readonly dataOffset: number;

    constructor(private readonly path: string) {
        const fd = openSync(path, "r");
        try {
            const prefix = Buffer.alloc(16);
            readSync(fd, prefix, 0, 16, 0);
            const headerPickleSize = prefix.readUInt32LE(4);
            const headerLength = prefix.readUInt32LE(12);
            const header = Buffer.alloc(headerLength);
            readSync(fd, header, 0, headerLength, 16);
            this.header = JSON.parse(header.toString("utf-8"));
            this.dataOffset = 8 + headerPickleSize;
        } finally {
            closeSync(fd);
        }
    }

    has(path: string) {
        return this.find(path) !== undefined;
    }

    list(path: string) {
        const entry = this.find(path);
        return entry && "files" in entry ? Object.keys(entry.files) : [];
    }

    read(path: string) {
        const entry = this.find(path);
        if (!entry || "files" in entry) {
            throw new Error(`${path} is not a file in ${this.path}`);
        }
        if (entry.unpacked) {
            return readFileSync(join(`${this.path}.unpacked`, path), "utf-8");
        }
        const buffer = Buffer.alloc(entry.size);
        const fd = openSync(this.path, "r");
        try {
            readSync(fd, buffer, 0, entry.size, this.dataOffset + Number(entry.offset));
        } finally {
            closeSync(fd);
        }
        return buffer.toString("utf-8");
    }

    private find(path: string) {
        let entry: AsarEntry | undefined = this.header;
        for (const segment of path.split("/").filter(Boolean)) {
            entry = entry && "files" in entry ? entry.files[segment] : undefined;
        }
        return entry;
    }
}

type AsarDirectory = { files: Record<string, AsarEntry> };
type AsarEntry = AsarDirectory | { size: number; offset: string; unpacked?: boolean };

main().catch((e) => {
    console.error(e instanceof Error ? e.message : e);
    process.exit(1);
});