Skip to content

Lightpanda: the headless browser designed for AI and automation

build: compile dependencies in ReleaseFast regardless of -Doptimize

Adrià Arrufat585cfb06ddaa
diff --git a/AGENTS.md b/AGENTS.md
index 53a2a07d2..f5f688947 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -6,6 +6,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to open a pull request (CLA, dev
Run `make download-v8` once first: it fetches the prebuilt V8 archive into `.lp-cache/`, which `build.zig` picks up automatically. Without it every build compiles V8 from source (10+ minutes).
+The C and Rust dependencies are built in ReleaseFast whatever `-Doptimize` is, so debug and release builds share them. Pass `ZIGFLAGS=-Ddebug_deps` to step into a dependency with a debugger.
+
```bash
make test # Run all tests
make test F="server" # Filter by substring
diff --git a/Makefile b/Makefile
index 5d3eb4369..a1466144c 100644
--- a/Makefile
+++ b/Makefile
@@ -3,8 +3,6 @@
ZIG := zig
BC := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-# tikv-jemalloc-sys's nested make can't parse inherited "-- F=..." overrides
-MAKEOVERRIDES =
# option test filter make test F="server"
F=
diff --git a/build.zig b/build.zig
index 42a58eba2..8b8b41ada 100644
--- a/build.zig
+++ b/build.zig
@@ -72,6 +72,15 @@ pub fn build(b: *Build) !void {
.glibc_version = devFastGlibcVersion(b),
}) else requested_target;
+ // Dependencies never follow -Doptimize, and they build for the requested
+ // target rather than the dev_fast bundled-CRT query, so debug and release
+ // builds share one set of dependency objects in the cache.
+ const debug_deps = b.option(bool, "debug_deps", "Build the C and Rust dependencies in Debug instead of ReleaseFast") orelse false;
+ const deps: Deps = .{
+ .target = requested_target,
+ .optimize = if (debug_deps) .Debug else .ReleaseFast,
+ };
+
// Without an explicit -Dprebuilt_v8_path, pick up whatever `make
// download-v8` cached rather than building V8 from source.
const prebuilt_v8_path = prebuilt_v8_path_option orelse if (enable_tsan or enable_asan) null else findPrebuiltV8(b, target, dev_fast);
@@ -120,12 +129,12 @@ pub fn build(b: *Build) !void {
const v8_archive: ?Build.LazyPath = if (prebuilt_v8_path) |path| .{ .cwd_relative = path } else null;
const v8_for_link = if (orderfile != null and v8_archive != null and !shared_v8) markHotSections(b, v8_archive.?) else v8_archive;
linkV8(b, lightpanda_module, enable_asan, enable_tsan, v8_for_link, shared_v8);
- linkCurl(b, lightpanda_module, enable_tsan, orderfile != null);
- linkRust(b, lightpanda_module);
+ linkCurl(b, lightpanda_module, deps, enable_tsan, orderfile != null);
+ linkRust(b, lightpanda_module, deps);
linkZenai(b, lightpanda_module);
linkIsocline(b, lightpanda_module);
- linkSqlite(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null);
- linkPcre2(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null);
+ linkSqlite(b, lightpanda_module, deps, enable_csan, enable_tsan, orderfile != null);
+ linkPcre2(b, lightpanda_module, deps, enable_csan, enable_tsan, orderfile != null);
// Check compilation
const check = b.step("check", "Check if lightpanda compiles");
@@ -211,6 +220,11 @@ pub fn build(b: *Build) !void {
}
}
+const Deps = struct {
+ target: Build.ResolvedTarget,
+ optimize: std.builtin.OptimizeMode,
+};
+
const ExeConfig = struct {
check: *Build.Step,
lightpanda_module: *Build.Module,
@@ -377,14 +391,14 @@ fn linkV8(
mod.addImport("v8", dep.module("v8"));
}
-fn linkRust(b: *Build, mod: *Build.Module) void {
- const is_debug = mod.optimize.? == .Debug;
+fn linkRust(b: *Build, mod: *Build.Module, deps: Deps) void {
+ // Cargo's "dev" profile writes to target/debug.
+ const profile, const out_subdir = if (deps.optimize == .Debug) .{ "dev", "debug" } else .{ "release", "release" };
// One cargo workspace, one staticlib (src/rust/Cargo.toml explains why).
const exec_cargo = b.addSystemCommand(&.{
"cargo", "build",
- "--profile", if (is_debug) "dev" else "release",
- "--features", if (is_debug) "memstats" else "",
+ "--profile", profile,
"--manifest-path", "src/rust/ffi/Cargo.toml",
});
@@ -397,17 +411,13 @@ fn linkRust(b: *Build, mod: *Build.Module) void {
// still surfaces the captured output.
_ = exec_cargo.captureStdErr(.{});
- // don't let cargo's progress report (sent to stderr) cause Zig's build to
- // print a 'failed command: ...' message. (non-zero status still outputs the error)
- _ = exec_cargo.captureStdErr(.{});
-
// TODO: We can prefer `--artifact-dir` once it become stable.
const out_dir = exec_cargo.addPrefixedOutputDirectoryArg("--target-dir=", "rust");
const rust_step = b.step("rust", "Build the Rust staticlib (requires cargo)");
rust_step.dependOn(&exec_cargo.step);
- const obj = out_dir.path(b, if (is_debug) "debug" else "release").path(b, "liblightpanda_ffi.a");
+ const obj = out_dir.path(b, out_subdir).path(b, "liblightpanda_ffi.a");
mod.addObjectFile(obj);
}
@@ -429,10 +439,10 @@ fn addDirInputs(b: *Build, run: *Build.Step.Run, root: []const u8, skip_dir: []c
}
}
-fn linkSqlite(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void {
+fn linkSqlite(b: *Build, mod: *Build.Module, deps: Deps, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void {
const dep = b.dependency("sqlite3", .{
- .target = mod.resolved_target.?,
- .optimize = mod.optimize.?,
+ .target = deps.target,
+ .optimize = deps.optimize,
});
const lib = sectionize(dep.artifact("sqlite3"), section);
@@ -487,10 +497,10 @@ fn linkSqlite(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is
mod.addImport("sqlite3", translate_c.createModule());
}
-fn linkPcre2(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void {
+fn linkPcre2(b: *Build, mod: *Build.Module, deps: Deps, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void {
const dep = b.dependency("pcre2", .{
- .target = mod.resolved_target.?,
- .optimize = mod.optimize.?,
+ .target = deps.target,
+ .optimize = deps.optimize,
.linkage = .static,
});
@@ -508,34 +518,32 @@ fn linkPcre2(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_
mod.addImport("pcre2", translate_c.createModule());
}
-fn linkCurl(b: *Build, mod: *Build.Module, is_tsan: bool, section: bool) void {
- const target = mod.resolved_target.?;
-
- const curl = buildCurl(b, target, mod.optimize.?, is_tsan, section);
+fn linkCurl(b: *Build, mod: *Build.Module, deps: Deps, is_tsan: bool, section: bool) void {
+ const curl = buildCurl(b, deps.target, deps.optimize, is_tsan, section);
mod.linkLibrary(curl);
const dep = b.dependency("curl", .{});
const translate_c = b.addTranslateC(.{
.root_source_file = dep.path("include/curl/curl.h"),
- .target = target,
+ .target = mod.resolved_target.?,
.optimize = mod.optimize.?,
});
translate_c.addIncludePath(dep.path("include"));
mod.addImport("curl", translate_c.createModule());
- const zlib = buildZlib(b, target, mod.optimize.?, is_tsan, section);
+ const zlib = buildZlib(b, deps.target, deps.optimize, is_tsan, section);
curl.root_module.linkLibrary(zlib);
- const brotli = buildBrotli(b, target, mod.optimize.?, is_tsan, section);
+ const brotli = buildBrotli(b, deps.target, deps.optimize, is_tsan, section);
for (brotli) |lib| curl.root_module.linkLibrary(lib);
- const nghttp2 = buildNghttp2(b, target, mod.optimize.?, is_tsan, section);
+ const nghttp2 = buildNghttp2(b, deps.target, deps.optimize, is_tsan, section);
curl.root_module.linkLibrary(nghttp2);
- const boringssl = buildBoringSsl(b, target, mod.optimize.?, section);
+ const boringssl = buildBoringSsl(b, deps.target, deps.optimize, section);
for (boringssl) |lib| curl.root_module.linkLibrary(lib);
- if (target.result.os.tag == .macos) {
+ if (deps.target.result.os.tag == .macos) {
// needed for proxying on mac
const framework_path = if (b.sysroot) |sysroot|
b.pathJoin(&.{ sysroot, "System/Library/Frameworks" })
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig
index 3b7cf3223..4d1a29776 100644
--- a/src/browser/parser/html5ever.zig
+++ b/src/browser/parser/html5ever.zig
@@ -96,13 +96,6 @@ pub extern "c" fn html5ever_parse_fragment(
pub extern "c" fn html5ever_attribute_iterator_next(ctx: *anyopaque) Nullable(Attribute);
pub extern "c" fn html5ever_attribute_iterator_count(ctx: *anyopaque) usize;
-pub extern "c" fn html5ever_get_memory_usage() MemoryUsage;
-
-const MemoryUsage = extern struct {
- resident: usize,
- allocated: usize,
-};
-
// Streaming parser API
pub extern "c" fn html5ever_streaming_parser_create(
doc: *anyopaque,
diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
index 455440f4f..4305378bf 100644
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@@ -52,16 +52,6 @@ dependencies = [
"syn 3.0.3",
]
-[[package]]
-name = "cc"
-version = "1.2.39"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f"
-dependencies = [
- "find-msvc-tools",
- "shlex",
-]
-
[[package]]
name = "cfg-if"
version = "1.0.0"
@@ -112,12 +102,6 @@ dependencies = [
"simd-adler32",
]
-[[package]]
-name = "find-msvc-tools"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959"
-
[[package]]
name = "flate2"
version = "1.1.9"
@@ -358,8 +342,6 @@ version = "0.1.0"
dependencies = [
"lightpanda-html5ever",
"lightpanda-render",
- "tikv-jemalloc-ctl",
- "tikv-jemallocator",
]
[[package]]
@@ -513,12 +495,6 @@ dependencies = [
"icu_properties",
]
-[[package]]
-name = "paste"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
-
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -668,12 +644,6 @@ dependencies = [
"syn 3.0.3",
]
-[[package]]
-name = "shlex"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
-
[[package]]
name = "simd-adler32"
version = "0.3.10"
@@ -782,37 +752,6 @@ dependencies = [
"utf-8",
]
-[[package]]
-name = "tikv-jemalloc-ctl"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c"
-dependencies = [
- "libc",
- "paste",
- "tikv-jemalloc-sys",
-]
-
-[[package]]
-name = "tikv-jemalloc-sys"
-version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b"
-dependencies = [
- "cc",
- "libc",
-]
-
-[[package]]
-name = "tikv-jemallocator"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a"
-dependencies = [
- "libc",
- "tikv-jemalloc-sys",
-]
-
[[package]]
name = "tiny-skia"
version = "0.12.0"
diff --git a/src/rust/ffi/Cargo.toml b/src/rust/ffi/Cargo.toml
index 9765ad866..3030ac768 100644
--- a/src/rust/ffi/Cargo.toml
+++ b/src/rust/ffi/Cargo.toml
@@ -11,8 +11,3 @@ crate-type = ["staticlib"]
[dependencies]
lightpanda-html5ever = { path = "../html5ever" }
lightpanda-render = { path = "../render" }
-tikv-jemallocator = { version = "0.6.1", features = ["stats"], optional = true }
-tikv-jemalloc-ctl = { version = "0.6.1", features = ["stats"], optional = true }
-
-[features]
-memstats = ["dep:tikv-jemallocator", "dep:tikv-jemalloc-ctl"]
diff --git a/src/rust/ffi/lib.rs b/src/rust/ffi/lib.rs
index f0b051be7..42d88da87 100644
--- a/src/rust/ffi/lib.rs
+++ b/src/rust/ffi/lib.rs
@@ -18,33 +18,7 @@
//! The one staticlib Zig links. Domain crates define their own `extern "C"`
//! entry points; `extern crate` is what pulls an otherwise-unreferenced crate
-//! into the archive (edition 2018+ drops unused `--extern` deps). Process-wide
-//! concerns (the allocator) live here, not in a domain crate.
+//! into the archive (edition 2018+ drops unused `--extern` deps).
extern crate lightpanda_html5ever;
extern crate lightpanda_render;
-
-#[cfg(feature = "memstats")]
-#[global_allocator]
-static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
-
-#[cfg(feature = "memstats")]
-#[repr(C)]
-pub struct Memory {
- pub resident: usize,
- pub allocated: usize,
-}
-
-#[cfg(feature = "memstats")]
-#[no_mangle]
-pub extern "C" fn html5ever_get_memory_usage() -> Memory {
- use tikv_jemalloc_ctl::{epoch, stats};
-
- // many statistics are cached and only updated when the epoch is advanced.
- let _ = epoch::advance();
-
- Memory {
- resident: stats::resident::read().unwrap_or(0),
- allocated: stats::allocated::read().unwrap_or(0),
- }
-}