Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
#!/usr/bin/env node
/**
* Reproducible source build of the Xapian/libzim WebAssembly runtime.
*
* Runs on any host with Docker (Windows, macOS, Linux). Everything compiles
* inside the pinned Emscripten image, so the host only needs Docker and Node.
* Git and network source downloads are needed only for the explicit
* --download-source fallback. No GPU is involved: this is a CPU-bound C++
* cross-compile, and it wants cores and RAM, not a graphics card.
*
* It deliberately does NOT use upstream's `libzim_release` target, which
* downloads a prebuilt libzim tarball. WebBrain cannot provide corresponding
* source for a binary it did not build. See docs/offline-rag-licensing.md.
*
* node scripts/build-zim-xapian.mjs # full build
* node scripts/build-zim-xapian.mjs --dry-run # validate setup, compile nothing
* node scripts/build-zim-xapian.mjs --keep # resume, reusing what already built
* node scripts/build-zim-xapian.mjs --work <dir> # build outside the repo
* node scripts/build-zim-xapian.mjs --link-opt 1 # pin the final optimizer level
* node scripts/build-zim-xapian.mjs --download-source # explicit network fallback
*
* Expect roughly 40-90 minutes on a fast machine: ICU and Xapian dominate.
*
* On Windows, prefer a --work directory on the WSL2 filesystem rather than a
* path under C:\. Docker bind mounts from NTFS stamp files with the host clock,
* which runs slightly ahead of the container and makes meson abort with "Clock
* skew detected", and they are far slower for builds with many small files.
*/
import { execFileSync, spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync, copyFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import path from 'node:path';
const buildDriverPath = fileURLToPath(import.meta.url);
const root = path.resolve(path.dirname(buildDriverPath), '..');
const dryRun = process.argv.includes('--dry-run');
const keepWorkTree = process.argv.includes('--keep');
const downloadSourceFallback = process.argv.includes('--download-source');
const workOverride = (() => {
const index = process.argv.indexOf('--work');
return index >= 0 && process.argv[index + 1] ? path.resolve(process.argv[index + 1]) : '';
})();
const linkOptOverride = (() => {
const index = process.argv.indexOf('--link-opt');
if (index < 0) return 0;
const level = Number(process.argv[index + 1]);
if (![1, 2, 3].includes(level)) fail('--link-opt takes 1, 2, or 3.');
return level;
})();
// Every version here must match the table in docs/offline-rag-licensing.md.
const PIN = Object.freeze({
repository: 'https://github.com/openzim/javascript-libzim.git',
tag: 'v0.95',
commit: '470b36920fba421a4c1a83b326e66d8aa0533870',
emscripten: '3.1.41',
libzim: '9.8.1',
xapian: '1.4.31',
xz: '5.2.6',
zlib: '1.3.1',
zstd: '1.5.7',
icu: '73.2',
});
// Only the WebAssembly build ships. The asm.js variant and the large-file test
// harness double the build time and are never loaded by the extension. The
// dependency build and the final link are separate steps so a failed link can
// be retried on its own: everything up to libzim.a takes minutes, the link
// takes seconds, and the link is the part that goes wrong.
const COMPILE_TARGETS = ['rename_pjsn', 'build/lib/libzim.a'];
const SHIPPED_ARTIFACTS = ['libzim-wasm.js', 'libzim-wasm.wasm'];
// wasm-opt runs last and walks the whole linked module, which for libzim plus
// Xapian plus ICU is a large one. It dies with SIGSEGV when it runs out of
// stack, so hand the container a big *finite* limit: glibc sizes worker-thread
// stacks from RLIMIT_STACK and falls back to a smaller default when that reads
// as unlimited. If it still crashes, step the optimizer down a level at a time
// rather than shipping nothing.
const LINK_STACK_BYTES = 536870912;
const LINK_OPT_LADDER = [3, 2, 1];
let linkOptimization = '';
const IMAGE_TAG = 'webbrain-emscripten-libzim:3.1.41';
const workTree = workOverride || path.join(root, '.build', 'zim-xapian');
const vendorDirectories = [
path.join(root, 'src', 'chrome', 'vendor', 'libzim'),
path.join(root, 'src', 'firefox', 'vendor', 'libzim'),
];
const correspondingSourceDir = path.join(root, 'dist', 'corresponding-source', `zim-xapian-${PIN.tag}`);
const archivedSourceBundle = existsSync(path.join(root, 'sbom.json'))
&& existsSync(path.join(root, 'Makefile'))
? root
: '';
const ARCHIVED_SOURCE_INPUTS = Object.freeze([
{ file: `icu4c-${PIN.icu.replace('.', '_')}-src.tgz` },
{ file: `libzim-${PIN.libzim}.tar.xz` },
{ file: `xapian-core-${PIN.xapian}.tar.xz` },
{ file: `xz-${PIN.xz}.tar.gz` },
{ file: `zlib-${PIN.zlib}.tar.gz` },
{ file: `zstd-${PIN.zstd}.tar.gz` },
{ file: 'Makefile' },
{ file: 'libzim_bindings.cpp' },
{ file: 'prejs_file_api.js' },
{ file: 'postjs_file_api.js' },
{ file: 'emscripten-crosscompile.ini' },
{ file: 'Dockerfile', target: path.join('docker', 'Dockerfile') },
{ file: 'LICENSE.javascript-libzim.txt', target: 'LICENSE' },
]);
function log(message) {
process.stdout.write(`${message}\n`);
}
function fail(message) {
process.stderr.write(`\nERROR: ${message}\n`);
process.exit(1);
}
function run(command, args, options = {}) {
log(` $ ${command} ${args.join(' ')}`);
if (dryRun && options.skipOnDryRun !== false) return '';
const result = spawnSync(command, args, { stdio: options.capture ? 'pipe' : 'inherit', cwd: options.cwd || root, encoding: 'utf8' });
if (result.error) fail(`${command} could not be started: ${result.error.message}`);
if (result.status !== 0) fail(`${command} exited with status ${result.status}.`);
return String(result.stdout || '');
}
function sha256(file) {
return createHash('sha256').update(readFileSync(file)).digest('hex');
}
// Inside WSL, Docker Desktop is installed on the Windows host but only reaches
// the distro when its WSL integration is switched on, so "not on PATH" here
// almost never means "not installed".
function insideWsl() {
if (process.platform !== 'linux') return false;
if (process.env.WSL_DISTRO_NAME) return true;
try { return /microsoft/i.test(readFileSync('/proc/version', 'utf8')); }
catch { return false; }
}
function dockerMissingHelp() {
if (!insideWsl()) {
return 'Docker is not on PATH. Install Docker Desktop (Windows/macOS) or docker-ce (Linux).';
}
return [
`Docker is not on PATH inside WSL${process.env.WSL_DISTRO_NAME ? ` (${process.env.WSL_DISTRO_NAME})` : ''}.`,
'Docker Desktop is probably running on Windows but not shared with this distro.',
'',
'In Docker Desktop: Settings -> Resources -> WSL Integration, enable this',
'distro, then Apply & Restart. Settings -> General must also have the WSL 2',
'based engine switched on.',
'',
'Open a new WSL shell afterwards, because the integration is added to PATH at',
'shell start, and confirm with: docker ps',
].join('\n');
}
function preflight() {
log('Preflight');
try {
const version = execFileSync('docker', ['--version'], { encoding: 'utf8' }).trim();
log(` docker: ${version}`);
} catch {
fail(dockerMissingHelp());
}
const daemon = spawnSync('docker', ['info', '--format', '{{.ServerVersion}}'], { encoding: 'utf8' });
if (daemon.status !== 0) {
fail(insideWsl()
? 'The Docker daemon is not reachable from WSL. Start Docker Desktop on Windows, and check Settings -> Resources -> WSL Integration for this distro.'
: 'The Docker daemon is not running. Start Docker Desktop and try again.');
}
log(` daemon: ${String(daemon.stdout || '').trim()}`);
if (!archivedSourceBundle || downloadSourceFallback) {
try {
execFileSync('git', ['--version'], { stdio: 'ignore' });
} catch {
fail('git is not on PATH. Install it to use the --download-source fallback.');
}
}
// The container writes as the invoking user on POSIX so build outputs are not
// left root-owned. Windows containers have no equivalent and need no mapping.
log(` host: ${process.platform}${process.platform === 'win32' ? ' (no uid mapping needed)' : ''}`);
}
function verifiedArchivedSourceInputs(directory) {
let sbom;
try {
sbom = JSON.parse(readFileSync(path.join(directory, 'sbom.json'), 'utf8'));
} catch (error) {
fail(`The archived corresponding-source SBOM is unreadable: ${error?.message || error}`);
}
if (sbom?.upstream?.commit !== PIN.commit) {
fail(`Archived source commit mismatch.\n expected ${PIN.commit}\n actual ${sbom?.upstream?.commit || '(missing)'}`);
}
const manifest = new Map((Array.isArray(sbom.correspondingSource) ? sbom.correspondingSource : [])
.map(record => [record?.file, record]));
return ARCHIVED_SOURCE_INPUTS.map((input) => {
const record = manifest.get(input.file);
const source = path.join(directory, input.file);
if (!record || !existsSync(source)) fail(`Archived build input is missing: ${input.file}`);
const bytes = statSync(source).size;
const digest = sha256(source);
if (bytes !== record.bytes || digest !== record.sha256) {
fail(`Archived build input failed SBOM verification: ${input.file}`);
}
return { ...input, source, bytes, sha256: digest };
});
}
function initializeArchivedWorkTree(inputs) {
mkdirSync(workTree, { recursive: true });
for (const input of inputs) {
const target = path.join(workTree, input.target || input.file);
mkdirSync(path.dirname(target), { recursive: true });
copyFileSync(input.source, target);
}
writeFileSync(path.join(workTree, 'package.json'), '{"private":true}\n');
mkdirSync(path.join(workTree, 'tests', 'prototype'), { recursive: true });
}
function verifyArchivedWorkTree(inputs) {
for (const input of inputs) {
const target = path.join(workTree, input.target || input.file);
if (!existsSync(target) || statSync(target).size !== input.bytes || sha256(target) !== input.sha256) {
fail(`Work-tree input differs from the archived corresponding source: ${input.target || input.file}`);
}
}
}
function guardIcuSourceDownload() {
const makefile = path.join(workTree, 'Makefile');
const contents = readFileSync(makefile, 'utf8');
const download = 'wget -N https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz';
const guarded = `[ -f icu4c-73_2-src.tgz ] || ${download}`;
if (contents.includes(guarded)) return;
if (!contents.includes(download)) fail('The ICU source download rule changed unexpectedly.');
writeFileSync(makefile, contents.replace(download, guarded));
}
function fetchSource() {
const useArchivedSource = Boolean(archivedSourceBundle) && !downloadSourceFallback;
const archivedInputs = useArchivedSource
? verifiedArchivedSourceInputs(archivedSourceBundle)
: null;
if (existsSync(workTree) && !keepWorkTree) {
// A fresh clone is the right default for a build whose output has to be
// reproducible, but say what is being thrown away: the dependency compile
// is the expensive part and --keep reuses it.
if (existsSync(path.join(workTree, 'build', 'lib', 'libzim.a'))) {
log(' the previous work tree holds a finished dependency build; --keep would reuse it');
}
log(` removing previous work tree ${workTree}`);
if (!dryRun) rmSync(workTree, { recursive: true, force: true });
}
if (!existsSync(workTree)) {
mkdirSync(path.dirname(workTree), { recursive: true });
if (useArchivedSource) {
log(` initializing from verified archived inputs in ${archivedSourceBundle}`);
if (!dryRun) initializeArchivedWorkTree(archivedInputs);
} else {
if (archivedSourceBundle) log(' --download-source selected; ignoring archived inputs and using the network fallback');
run('git', ['clone', '--branch', PIN.tag, '--depth', '1', PIN.repository, workTree]);
}
} else {
log(` reusing ${workTree}`);
}
if (dryRun) return;
if (useArchivedSource) {
verifyArchivedWorkTree(archivedInputs);
guardIcuSourceDownload();
log(` archived inputs verified for commit ${PIN.commit}`);
return;
}
const head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: workTree, encoding: 'utf8' }).trim();
if (head !== PIN.commit) {
fail(`Pinned commit mismatch.\n expected ${PIN.commit}\n actual ${head}\nRefusing to build an unpinned tree.`);
}
log(` commit verified: ${head}`);
guardIcuSourceDownload();
}
function dockerMount(hostPath) {
// Docker Desktop on Windows accepts a drive-letter path; POSIX passes through.
return process.platform === 'win32' ? hostPath.replace(/\\/g, '/') : hostPath;
}
// meson aborts on any timestamp it reads as being in the future, and a Docker
// bind mount from NTFS stamps files with the Windows clock while the container
// runs its own slightly behind. That surfaces two dependencies deep as an
// opaque "Clock skew detected", so measure it up front against the real mount.
function checkClockSkew() {
log('\nChecking mount clock skew');
const probe = path.join(workTree, '.clock-probe');
writeFileSync(probe, 'probe');
const hostSeconds = Math.floor(statSync(probe).mtimeMs / 1000);
const args = ['run', '--rm', '-v', `${dockerMount(workTree)}:/src`, IMAGE_TAG,
'bash', '-c', 'echo "$(date +%s) $(stat -c %Y /src/.clock-probe)"'];
const result = spawnSync('docker', args, { encoding: 'utf8' });
rmSync(probe, { force: true });
if (result.status !== 0) {
log(' could not measure skew; continuing');
return;
}
const [containerNow, mountStamp] = String(result.stdout || '').trim().split(/\s+/).map(Number);
if (!Number.isFinite(containerNow) || !Number.isFinite(mountStamp)) {
log(' could not measure skew; continuing');
return;
}
const ahead = mountStamp - containerNow;
log(` host stamp ${hostSeconds}, container now ${containerNow}, mount is ${ahead}s ahead`);
if (ahead <= 0) {
log(' ok');
return;
}
fail([
`The mounted filesystem is ${ahead}s ahead of the container clock.`,
'meson refuses to build against a future timestamp and will abort on zstd.',
'',
'Fix:',
' 1. Build on the WSL2 filesystem rather than a path under C:\\. This removes',
' the skew for good and is markedly faster, because an NTFS bind mount is',
' slow for a build made of many small files. From inside a WSL shell:',
' npm run build:zim-xapian -- --work ~/zim-xapian-build',
' 2. Or restart Docker Desktop ("wsl --shutdown", then start it) to resync the',
' VM clock, and rerun with --keep. Quicker to try, but the drift returns',
' after the host sleeps.',
].join('\n'));
}
function buildImage() {
log('\nBuilding the Emscripten image');
run('docker', ['build', '-t', IMAGE_TAG, '--build-arg', `VERSION=${PIN.emscripten}`, path.join(workTree, 'docker')]);
}
function dockerRun(command, { env = {}, allowFailure = false } = {}) {
const args = ['run', '--rm', '-v', `${dockerMount(workTree)}:/src`,
'--ulimit', `stack=${LINK_STACK_BYTES}:${LINK_STACK_BYTES}`];
if (process.platform !== 'win32') {
const uid = typeof process.getuid === 'function' ? process.getuid() : 0;
const gid = typeof process.getgid === 'function' ? process.getgid() : 0;
args.push('-u', `${uid}:${gid}`);
}
for (const [name, value] of Object.entries(env)) args.push('-e', `${name}=${value}`);
args.push(IMAGE_TAG, ...command);
if (!allowFailure) {
run('docker', args);
return 0;
}
log(` $ docker ${args.join(' ')}`);
const result = spawnSync('docker', args, { stdio: 'inherit', cwd: root });
if (result.error) fail(`docker could not be started: ${result.error.message}`);
return result.status === 0 ? 0 : (result.status ?? 1);
}
// The Makefile moves package.json aside while libzim builds and moves it back
// afterwards, because Emscripten trips over it. A build that dies in between
// leaves only package.json.temp, and the next run's `mv package.json` then
// fails on a missing file, so a resume has to put the tree back first.
function normalizeWorkTree() {
const live = path.join(workTree, 'package.json');
const stashed = path.join(workTree, 'package.json.temp');
if (!existsSync(stashed) || existsSync(live)) return;
log(' restoring package.json stashed by an interrupted build');
copyFileSync(stashed, live);
rmSync(stashed, { force: true });
}
function compile() {
log('\nCompiling from source (ICU, Xapian, and libzim dominate this step)');
normalizeWorkTree();
dockerRun(['make', ...COMPILE_TARGETS]);
linkRuntime();
dockerRun(['make', 'restore_pjsn']);
}
// EMCC_CFLAGS is appended to the compiler's own argument list, so it overrides
// the -O3 baked into the upstream recipe without patching the Makefile.
function linkRuntime() {
const ladder = linkOptOverride ? [linkOptOverride] : LINK_OPT_LADDER;
for (const level of ladder) {
log(`\nLinking the runtime at -O${level}`);
// make treats a half-written artifact from the last attempt as up to date.
for (const name of SHIPPED_ARTIFACTS) rmSync(path.join(workTree, name), { force: true });
const status = dockerRun(['make', 'libzim-wasm.js'], { env: { EMCC_CFLAGS: `-O${level}` }, allowFailure: true });
if (status === 0) {
linkOptimization = `O${level}`;
log(` linked at -O${level}`);
return;
}
if (level !== ladder[ladder.length - 1]) {
log(`\n The -O${level} link failed. Retrying one optimizer level lower.`);
}
}
fail(linkFailureHelp());
}
function linkFailureHelp() {
return [
'Linking libzim-wasm.js failed at every optimizer level tried.',
'',
'Check what the last error actually was. "wasm-opt ... failed (received',
'SIGSEGV)" means Emscripten\'s optimizer ran out of room, not that anything',
'is wrong with the code; a compiler or linker error means something else.',
'',
'The compiled libraries under build/ survive, so each retry only repeats the',
'final link, which takes seconds:',
'',
' 1. Give the Docker VM more memory. Under WSL2 it takes half the host by',
' default: put "memory=16GB" under [wsl2] in %UserProfile%\\.wslconfig,',
' run "wsl --shutdown", restart Docker Desktop, and rerun with --keep.',
' 2. Rerun with --keep --link-opt 1 to pin the lightest optimizer pipeline.',
].join('\n');
}
function collectLicenseFiles() {
const found = [];
const candidates = [
['javascript-libzim', path.join(workTree, 'LICENSE')],
['libzim', path.join(workTree, `libzim-${PIN.libzim}`, 'COPYING')],
['xapian-core', path.join(workTree, `xapian-core-${PIN.xapian}`, 'COPYING')],
['xz', path.join(workTree, `xz-${PIN.xz}`, 'COPYING')],
['zlib', path.join(workTree, `zlib-${PIN.zlib}`, 'LICENSE')],
['zstd', path.join(workTree, `zstd-${PIN.zstd}`, 'LICENSE')],
['icu', path.join(workTree, 'icu', 'LICENSE')],
];
for (const [name, file] of candidates) {
if (existsSync(file)) found.push({ name, file });
}
return found;
}
function archiveCorrespondingSource() {
log('\nArchiving corresponding source');
rmSync(correspondingSourceDir, { recursive: true, force: true });
mkdirSync(correspondingSourceDir, { recursive: true });
const manifest = [];
const tarballs = readdirSync(workTree).filter(name => /\.(tar\.(gz|xz)|tgz)$/.test(name));
if (!tarballs.length && !dryRun) {
fail('No source tarballs found in the work tree. The build did not run from source.');
}
for (const name of tarballs) {
const from = path.join(workTree, name);
const to = path.join(correspondingSourceDir, name);
copyFileSync(from, to);
manifest.push({ file: name, bytes: statSync(to).size, sha256: sha256(to) });
log(` ${name} (${statSync(to).size} bytes)`);
}
// The Makefile patches libzim in place with sed. Ship the recipe that produced
// those edits, because the patched tree is what the binary was built from.
const makefile = path.join(workTree, 'Makefile');
if (existsSync(makefile)) {
copyFileSync(makefile, path.join(correspondingSourceDir, 'Makefile'));
manifest.push({ file: 'Makefile', bytes: statSync(makefile).size, sha256: sha256(makefile), note: 'contains the in-place libzim patches applied before compilation' });
}
for (const [name, file] of [['libzim_bindings.cpp', path.join(workTree, 'libzim_bindings.cpp')],
['prejs_file_api.js', path.join(workTree, 'prejs_file_api.js')],
['postjs_file_api.js', path.join(workTree, 'postjs_file_api.js')],
['Dockerfile', path.join(workTree, 'docker', 'Dockerfile')],
['emscripten-crosscompile.ini', path.join(workTree, 'emscripten-crosscompile.ini')]]) {
if (!existsSync(file)) continue;
copyFileSync(file, path.join(correspondingSourceDir, name));
manifest.push({ file: name, bytes: statSync(file).size, sha256: sha256(file) });
}
for (const { name, file } of collectLicenseFiles()) {
const to = path.join(correspondingSourceDir, `LICENSE.${name}.txt`);
copyFileSync(file, to);
manifest.push({ file: path.basename(to), bytes: statSync(to).size, sha256: sha256(to) });
}
const archivedDriver = path.join(correspondingSourceDir, 'scripts', 'build-zim-xapian.mjs');
mkdirSync(path.dirname(archivedDriver), { recursive: true });
copyFileSync(buildDriverPath, archivedDriver);
manifest.push({
file: 'scripts/build-zim-xapian.mjs',
bytes: statSync(archivedDriver).size,
sha256: sha256(archivedDriver),
note: 'exact Docker build driver used to produce the shipped WebAssembly runtime',
});
const rebuildGuide = path.join(correspondingSourceDir, 'README.md');
writeFileSync(rebuildGuide, correspondingSourceReadme());
manifest.push({
file: 'README.md',
bytes: statSync(rebuildGuide).size,
sha256: sha256(rebuildGuide),
note: 'standalone prerequisites and rebuild commands',
});
return manifest;
}
function correspondingSourceReadme() {
return `# Xapian/libzim WebAssembly corresponding source
This directory contains the source archives, license texts, build inputs, and
the exact WebBrain driver used to produce the bundled runtime. The build pins
javascript-libzim ${PIN.tag} at commit
\`${PIN.commit}\` and Emscripten ${PIN.emscripten}.
## Rebuild
Install Docker and Node.js. From the root of this extracted directory run:
\`\`\`sh
node scripts/build-zim-xapian.mjs --work .build/zim-xapian
\`\`\`
On Windows, run from WSL2 and place \`--work\` on the WSL filesystem to avoid
Docker bind-mount clock skew. By default the driver verifies the files beside
it against \`sbom.json\` and copies those archived inputs into the work tree, so
the published source archive—not a fresh network clone—is what gets built. It
then builds the pinned Emscripten image, compiles libzim/Xapian and dependencies
from source, retries the final Wasm optimizer down to \`-O1\` if necessary, and
writes the runtime plus refreshed SBOM records under
\`src/{chrome,firefox}/vendor/libzim/\`.
If the archived inputs are deliberately unavailable, install Git and explicitly
allow the upstream/dependency download fallback:
\`\`\`sh
node scripts/build-zim-xapian.mjs --work .build/zim-xapian --download-source
\`\`\`
Inside a full WebBrain checkout, \`npm run build:zim-xapian\` invokes the same
driver. The source archives in this directory are the immutable default inputs
retained with the release; their sizes and SHA-256 hashes are recorded in
\`sbom.json\`.
`;
}
function vendorArtifacts() {
log('\nVendoring build outputs');
const artifacts = [];
for (const name of SHIPPED_ARTIFACTS) {
const produced = path.join(workTree, name);
if (!existsSync(produced)) {
if (dryRun) { log(` (dry run) would expect ${name}`); continue; }
fail(`The build did not produce ${name}.`);
}
const digest = sha256(produced);
const bytes = statSync(produced).size;
artifacts.push({ file: name, bytes, sha256: digest });
for (const directory of vendorDirectories) {
mkdirSync(directory, { recursive: true });
copyFileSync(produced, path.join(directory, name));
}
log(` ${name} ${bytes} bytes sha256 ${digest}`);
}
for (const { name, file } of collectLicenseFiles()) {
for (const directory of vendorDirectories) {
mkdirSync(directory, { recursive: true });
copyFileSync(file, path.join(directory, `LICENSE.${name}.txt`));
}
}
return artifacts;
}
function writeRecords(artifacts, correspondingSource) {
const sbom = {
component: 'zim-xapian-wasm',
builtAt: new Date().toISOString(),
builtFrom: 'source',
upstream: { repository: PIN.repository, tag: PIN.tag, commit: PIN.commit },
toolchain: { emscripten: PIN.emscripten, image: IMAGE_TAG, linkOptimization: linkOptimization || `O${LINK_OPT_LADDER[0]}` },
dependencies: [
{ name: 'javascript-libzim', version: PIN.tag, license: 'GPL-3.0-or-later' },
{ name: 'libzim', version: PIN.libzim, license: 'GPL-2.0-or-later' },
{ name: 'xapian-core', version: PIN.xapian, license: 'GPL-2.0-or-later' },
{ name: 'xz', version: PIN.xz, license: 'public-domain with per-file exceptions' },
{ name: 'zlib', version: PIN.zlib, license: 'Zlib' },
{ name: 'zstd', version: PIN.zstd, license: 'BSD-3-Clause OR GPL-2.0' },
{ name: 'icu', version: PIN.icu, license: 'Unicode-DFS-2016' },
],
artifacts,
correspondingSource,
effectiveReleaseLicense: 'GPL-3.0-or-later',
};
if (dryRun) { log('\n(dry run) SBOM not written'); return; }
for (const directory of vendorDirectories) {
mkdirSync(directory, { recursive: true });
writeFileSync(path.join(directory, 'sbom.json'), `${JSON.stringify(sbom, null, 2)}\n`);
writeFileSync(path.join(directory, 'README.webbrain.md'), vendorReadme(artifacts));
}
writeFileSync(path.join(correspondingSourceDir, 'sbom.json'), `${JSON.stringify(sbom, null, 2)}\n`);
}
function vendorReadme(artifacts) {
const rows = artifacts.map(item => `- \`${item.file}\`: ${item.bytes} bytes, SHA-256 \`${item.sha256}\``).join('\n');
const fallback = linkOptimization && linkOptimization !== `O${LINK_OPT_LADDER[0]}`
? `\n\nLinked at \`-${linkOptimization}\` rather than \`-O${LINK_OPT_LADDER[0]}\`, because Emscripten's wasm-opt\ncould not optimize this module at the higher level. The module is correct and\nsomewhat larger.`
: '';
return `# Vendored Xapian/libzim WebAssembly runtime
Built from source by \`scripts/build-zim-xapian.mjs\`. Do not hand-copy an
upstream release asset here: WebBrain cannot provide corresponding source for a
binary it did not build. See \`docs/offline-rag-licensing.md\`.
- Upstream: ${PIN.repository} \`${PIN.tag}\` (\`${PIN.commit}\`)
- Toolchain: Emscripten ${PIN.emscripten}
- libzim ${PIN.libzim}, Xapian ${PIN.xapian}, ICU ${PIN.icu}, zstd ${PIN.zstd}, xz ${PIN.xz}, zlib ${PIN.zlib}
${rows}${fallback}
## License
This runtime is GPL. Any release artifact that bundles it is conveyed under
**GPL-3.0-or-later**. WebBrain 33.0.0 and later uses that license because the
distributed extension integrates this runtime. Complete corresponding source for these
binaries is published as a \`webbrain-zim-xapian-*-corresponding-source.zip\`
release asset and must accompany every release.
To rebuild: \`npm run build:zim-xapian\`
`;
}
log(`Xapian/libzim runtime build${dryRun ? ' (dry run)' : ''}\n`);
preflight();
log('\nFetching pinned source');
fetchSource();
if (!dryRun) {
buildImage();
checkClockSkew();
compile();
}
const artifacts = vendorArtifacts();
const correspondingSource = dryRun ? [] : archiveCorrespondingSource();
writeRecords(artifacts, correspondingSource);
log(`\n${dryRun ? 'Dry run complete. Setup looks usable.' : 'Build complete.'}`);
if (!dryRun) {
log('\nNext steps:');
log(' 1. Commit src/{chrome,firefox}/vendor/libzim/ and dist/corresponding-source/.');
log(' 2. Flip ZIM_XAPIAN_RUNTIME_BUNDLED to true in both zim-xapian.js files.');
log(' 3. Run npm test.');
}