SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
release: ship the Linux Rust binaries as static musl builds (#12284)
Chris Lu42b6bc3d2620
diff --git a/.github/actions/rust-musl-target/action.yml b/.github/actions/rust-musl-target/action.ymlnew file mode 100644index 0000000000..2acc844fd5--- /dev/null+++ b/.github/actions/rust-musl-target/action.yml@@ -0,0 +1,57 @@+name: Set up a musl Rust target+description: Install the stable toolchain for a static Linux musl target, with the C toolchain and the link shims aws-lc-sys needs.+inputs:+ target:+ description: x86_64-unknown-linux-musl or aarch64-unknown-linux-musl+ required: true+runs:+ using: composite+ steps:+ - name: Install Rust toolchain+ uses: dtolnay/rust-toolchain@stable+ with:+ targets: ${{ inputs.target }}++ - name: Install musl tools (amd64)+ if: inputs.target == 'x86_64-unknown-linux-musl'+ shell: bash+ run: |+ sudo apt-get install -y musl-tools+ # C deps (aws-lc-sys) must compile against musl headers, not the host+ # gcc's glibc headers — glibc's C23 redirects (__isoc23_sscanf,+ # __isoc23_strtol) don't exist in musl and fail the link.+ echo "CC_x86_64_unknown_linux_musl=musl-gcc" >> "$GITHUB_ENV"++ - name: Install cross-compilation tools (arm64)+ if: inputs.target == 'aarch64-unknown-linux-musl'+ shell: bash+ run: |+ sudo apt-get install -y gcc-aarch64-linux-gnu+ echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"+ # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl+ echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"+ # aws-lc-sys compiles against glibc headers, which redirect+ # sscanf/strtol to C23 __isoc23_* symbols musl does not provide.+ # Link a tiny forwarding shim so those references resolve to the+ # musl implementations. Prototypes are declared manually: including+ # glibc headers here would re-redirect the calls back to __isoc23_*.+ cat > /tmp/isoc23-shim.c <<'SHIM'+ #include <stdarg.h>+ extern int vsscanf(const char *s, const char *fmt, va_list ap);+ extern long strtol(const char *nptr, char **endptr, int base);+ int __isoc23_sscanf(const char *s, const char *fmt, ...) {+ va_list ap;+ va_start(ap, fmt);+ int r = vsscanf(s, fmt, ap);+ va_end(ap);+ return r;+ }+ long __isoc23_strtol(const char *nptr, char **endptr, int base) {+ return strtol(nptr, endptr, base);+ }+ SHIM+ aarch64-linux-gnu-gcc -O2 -c /tmp/isoc23-shim.c -o /tmp/isoc23-shim.o+ # link-args land after libc on the link line; rescan musl libc so+ # the shim's own vsscanf/strtol references resolve, and libgcc for+ # the fp128 builtins musl's vfscanf/floatscan pull in on aarch64.+ echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=-Clink-arg=/tmp/isoc23-shim.o -Clink-arg=-lc -Clink-arg=-lgcc" >> "$GITHUB_ENV"diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.ymlindex c29773dd6d..45619a21e0 100644--- a/.github/workflows/container_dev.yml+++ b/.github/workflows/container_dev.yml@@ -32,57 +32,14 @@ jobs:arch: amd64- target: aarch64-unknown-linux-muslarch: arm64- cross: truesteps:- name: Checkoutuses: actions/checkout@v7- - name: Install Rust toolchain- uses: dtolnay/rust-toolchain@stable+ - name: Set up the musl target+ uses: ./.github/actions/rust-musl-targetwith:- targets: ${{ matrix.target }}-- - name: Install musl tools (amd64)- if: ${{ !matrix.cross }}- run: |- sudo apt-get install -y musl-tools- # C deps (aws-lc-sys) must compile against musl headers, not the host- # gcc's glibc headers — glibc's C23 redirects (__isoc23_sscanf,- # __isoc23_strtol) don't exist in musl and fail the link.- echo "CC_x86_64_unknown_linux_musl=musl-gcc" >> "$GITHUB_ENV"-- - name: Install cross-compilation tools (arm64)- if: matrix.cross- run: |- sudo apt-get install -y gcc-aarch64-linux-gnu- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"- # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl- echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"- # aws-lc-sys compiles against glibc headers, which redirect- # sscanf/strtol to C23 __isoc23_* symbols musl does not provide.- # Link a tiny forwarding shim so those references resolve to the- # musl implementations. Prototypes are declared manually: including- # glibc headers here would re-redirect the calls back to __isoc23_*.- cat > /tmp/isoc23-shim.c <<'SHIM'- #include <stdarg.h>- extern int vsscanf(const char *s, const char *fmt, va_list ap);- extern long strtol(const char *nptr, char **endptr, int base);- int __isoc23_sscanf(const char *s, const char *fmt, ...) {- va_list ap;- va_start(ap, fmt);- int r = vsscanf(s, fmt, ap);- va_end(ap);- return r;- }- long __isoc23_strtol(const char *nptr, char **endptr, int base) {- return strtol(nptr, endptr, base);- }- SHIM- aarch64-linux-gnu-gcc -O2 -c /tmp/isoc23-shim.c -o /tmp/isoc23-shim.o- # link-args land after libc on the link line; rescan musl libc so- # the shim's own vsscanf/strtol references resolve, and libgcc for- # the fp128 builtins musl's vfscanf/floatscan pull in on aarch64.- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=-Clink-arg=/tmp/isoc23-shim.o -Clink-arg=-lc -Clink-arg=-lgcc" >> "$GITHUB_ENV"+ target: ${{ matrix.target }}- name: Cache cargo registry and targetif: runner.environment == 'github-hosted'diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.ymlindex 5f2485f4ee..668c6046f6 100644--- a/.github/workflows/container_latest.yml+++ b/.github/workflows/container_latest.yml@@ -74,59 +74,18 @@ jobs:arch: amd64- target: aarch64-unknown-linux-muslarch: arm64- cross: truesteps:- name: Checkoutuses: actions/checkout@v7with:ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}- - name: Install Rust toolchain- uses: dtolnay/rust-toolchain@stable+ # The checkout above is the source ref, which may predate these+ # actions; take them from master, the ref this fallback runs from.+ - name: Set up the musl target+ uses: seaweedfs/seaweedfs/.github/actions/rust-musl-target@masterwith:- targets: ${{ matrix.target }}-- - name: Install musl tools (amd64)- if: ${{ !matrix.cross }}- run: |- sudo apt-get install -y musl-tools- # C deps (aws-lc-sys) must compile against musl headers, not the host- # gcc's glibc headers — glibc's C23 redirects (__isoc23_sscanf,- # __isoc23_strtol) don't exist in musl and fail the link.- echo "CC_x86_64_unknown_linux_musl=musl-gcc" >> "$GITHUB_ENV"-- - name: Install cross-compilation tools (arm64)- if: matrix.cross- run: |- sudo apt-get install -y gcc-aarch64-linux-gnu- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"- # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl- echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"- # aws-lc-sys compiles against glibc headers, which redirect- # sscanf/strtol to C23 __isoc23_* symbols musl does not provide.- # Link a tiny forwarding shim so those references resolve to the- # musl implementations. Prototypes are declared manually: including- # glibc headers here would re-redirect the calls back to __isoc23_*.- cat > /tmp/isoc23-shim.c <<'SHIM'- #include <stdarg.h>- extern int vsscanf(const char *s, const char *fmt, va_list ap);- extern long strtol(const char *nptr, char **endptr, int base);- int __isoc23_sscanf(const char *s, const char *fmt, ...) {- va_list ap;- va_start(ap, fmt);- int r = vsscanf(s, fmt, ap);- va_end(ap);- return r;- }- long __isoc23_strtol(const char *nptr, char **endptr, int base) {- return strtol(nptr, endptr, base);- }- SHIM- aarch64-linux-gnu-gcc -O2 -c /tmp/isoc23-shim.c -o /tmp/isoc23-shim.o- # link-args land after libc on the link line; rescan musl libc so- # the shim's own vsscanf/strtol references resolve, and libgcc for- # the fp128 builtins musl's vfscanf/floatscan pull in on aarch64.- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=-Clink-arg=/tmp/isoc23-shim.o -Clink-arg=-lc -Clink-arg=-lgcc" >> "$GITHUB_ENV"+ target: ${{ matrix.target }}- name: Cache cargo registry and targetif: runner.environment == 'github-hosted'@@ -201,7 +160,7 @@ jobs:with:ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}- name: Clean up stale FUSE mounts- uses: ./.github/actions/cleanup-stale-fuse-mounts+ uses: seaweedfs/seaweedfs/.github/actions/cleanup-stale-fuse-mounts@master- name: Free Disk Spacerun: |diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.ymlindex 7f611943f4..c78e5da7dc 100644--- a/.github/workflows/container_release_unified.yml+++ b/.github/workflows/container_release_unified.yml@@ -55,57 +55,14 @@ jobs:arch: amd64- target: aarch64-unknown-linux-muslarch: arm64- cross: truesteps:- name: Checkoutuses: actions/checkout@v7- - name: Install Rust toolchain- uses: dtolnay/rust-toolchain@stable+ - name: Set up the musl target+ uses: ./.github/actions/rust-musl-targetwith:- targets: ${{ matrix.target }}-- - name: Install musl tools (amd64)- if: ${{ !matrix.cross }}- run: |- sudo apt-get install -y musl-tools- # C deps (aws-lc-sys) must compile against musl headers, not the host- # gcc's glibc headers — glibc's C23 redirects (__isoc23_sscanf,- # __isoc23_strtol) don't exist in musl and fail the link.- echo "CC_x86_64_unknown_linux_musl=musl-gcc" >> "$GITHUB_ENV"-- - name: Install cross-compilation tools (arm64)- if: matrix.cross- run: |- sudo apt-get install -y gcc-aarch64-linux-gnu- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"- # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl- echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"- # aws-lc-sys compiles against glibc headers, which redirect- # sscanf/strtol to C23 __isoc23_* symbols musl does not provide.- # Link a tiny forwarding shim so those references resolve to the- # musl implementations. Prototypes are declared manually: including- # glibc headers here would re-redirect the calls back to __isoc23_*.- cat > /tmp/isoc23-shim.c <<'SHIM'- #include <stdarg.h>- extern int vsscanf(const char *s, const char *fmt, va_list ap);- extern long strtol(const char *nptr, char **endptr, int base);- int __isoc23_sscanf(const char *s, const char *fmt, ...) {- va_list ap;- va_start(ap, fmt);- int r = vsscanf(s, fmt, ap);- va_end(ap);- return r;- }- long __isoc23_strtol(const char *nptr, char **endptr, int base) {- return strtol(nptr, endptr, base);- }- SHIM- aarch64-linux-gnu-gcc -O2 -c /tmp/isoc23-shim.c -o /tmp/isoc23-shim.o- # link-args land after libc on the link line; rescan musl libc so- # the shim's own vsscanf/strtol references resolve, and libgcc for- # the fp128 builtins musl's vfscanf/floatscan pull in on aarch64.- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=-Clink-arg=/tmp/isoc23-shim.o -Clink-arg=-lc -Clink-arg=-lgcc" >> "$GITHUB_ENV"+ target: ${{ matrix.target }}- name: Cache cargo registry and targetif: runner.environment == 'github-hosted'diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.ymlindex 8ddde92b05..3999c6df34 100644--- a/.github/workflows/rust_binaries_release.yml+++ b/.github/workflows/rust_binaries_release.yml@@ -19,37 +19,20 @@ jobs:strategy:matrix:include:- - target: x86_64-unknown-linux-gnu+ # Static musl, as the container images ship: a glibc build only+ # starts on distributions at least as new as the runner's.+ - target: x86_64-unknown-linux-muslasset_suffix: linux_amd64- - target: aarch64-unknown-linux-gnu+ - target: aarch64-unknown-linux-muslasset_suffix: linux_arm64- cross: truesteps:- uses: actions/checkout@v7- - name: Install Rust toolchain- uses: dtolnay/rust-toolchain@stable+ - name: Set up the musl target+ uses: ./.github/actions/rust-musl-targetwith:- targets: ${{ matrix.target }}-- - name: Install cross-compilation tools- if: matrix.cross- run: |- sudo dpkg --add-architecture arm64- # noble keeps apt sources in deb822 ubuntu.sources, not sources.list —- # pin them to amd64 so apt does not probe arm64 indexes on- # archive/security.ubuntu.com (they 404).- sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources- for suite in noble noble-updates noble-security; do- echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list- done- sudo apt-get update- sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"- echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"- echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"- echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"+ target: ${{ matrix.target }}- name: Cache cargo registry and targetif: runner.environment == 'github-hosted'@@ -128,11 +111,12 @@ jobs:strategy:matrix:include:- - target: x86_64-unknown-linux-gnu+ # Static musl, as the container images ship: a glibc build only+ # starts on distributions at least as new as the runner's.+ - target: x86_64-unknown-linux-muslasset_suffix: linux_amd64- - target: aarch64-unknown-linux-gnu+ - target: aarch64-unknown-linux-muslasset_suffix: linux_arm64- cross: truesteps:- uses: actions/checkout@v7@@ -141,25 +125,10 @@ jobs:# should not find another one sitting in the checkout's git config.persist-credentials: false- - name: Install Rust toolchain- uses: dtolnay/rust-toolchain@stable+ - name: Set up the musl target+ uses: ./.github/actions/rust-musl-targetwith:- targets: ${{ matrix.target }}-- - name: Install cross-compilation tools- if: matrix.cross- run: |- sudo dpkg --add-architecture arm64- # noble keeps apt sources in deb822 ubuntu.sources, not sources.list —- # pin them to amd64 so apt does not probe arm64 indexes on- # archive/security.ubuntu.com (they 404).- sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources- for suite in noble noble-updates noble-security; do- echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list- done- sudo apt-get update- sudo apt-get install -y gcc-aarch64-linux-gnu- echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"+ target: ${{ matrix.target }}- name: Cache cargo registry and targetif: runner.environment == 'github-hosted'