Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

Build the Rust worker against the protoc that ships with the build (#10881)

Chris Lu7ebf2ebac3ab
diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml
index 1800de6859..2aa5a702b0 100644
--- a/.github/workflows/container_dev.yml
+++ b/.github/workflows/container_dev.yml
@@ -49,11 +49,6 @@ jobs:
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- # lance's build scripts compile their own protos, and unlike seaweed-volume
- # they do not vendor a protoc to do it with.
- - name: Install protoc
- run: sudo apt-get install -y protobuf-compiler
-
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
@@ -66,6 +61,21 @@ jobs:
restore-keys: |
rust-docker-dev-${{ matrix.target }}-
+ # lance's build scripts compile their own protos and look for a protoc.
+ # Point them at the one protoc-bin-vendored ships, which seaweed-worker's
+ # own build already uses, so no job depends on a system package and every
+ # build sees the same version.
+ - name: Use the vendored protoc
+ run: |
+ cd seaweed-worker
+ cargo fetch
+ # The version from the lock, not whatever else a restored cache holds.
+ version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
+ test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
+ protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
+ test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
+ echo "PROTOC=$protoc" >> "$GITHUB_ENV"
+
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml
index 8ef7eaf4c1..eadb69742e 100644
--- a/.github/workflows/container_latest.yml
+++ b/.github/workflows/container_latest.yml
@@ -93,11 +93,6 @@ jobs:
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- # lance's build scripts compile their own protos, and unlike seaweed-volume
- # they do not vendor a protoc to do it with.
- - name: Install protoc
- run: sudo apt-get install -y protobuf-compiler
-
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
@@ -110,6 +105,21 @@ jobs:
restore-keys: |
rust-docker-${{ matrix.target }}-
+ # lance's build scripts compile their own protos and look for a protoc.
+ # Point them at the one protoc-bin-vendored ships, which seaweed-worker's
+ # own build already uses, so no job depends on a system package and every
+ # build sees the same version.
+ - name: Use the vendored protoc
+ run: |
+ cd seaweed-worker
+ cargo fetch
+ # The version from the lock, not whatever else a restored cache holds.
+ version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
+ test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
+ protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
+ test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
+ echo "PROTOC=$protoc" >> "$GITHUB_ENV"
+
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml
index 7a55bad304..63bae88573 100644
--- a/.github/workflows/container_release_unified.yml
+++ b/.github/workflows/container_release_unified.yml
@@ -77,11 +77,6 @@ jobs:
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- # lance's build scripts compile their own protos, and unlike seaweed-volume
- # they do not vendor a protoc to do it with.
- - name: Install protoc
- run: sudo apt-get install -y protobuf-compiler
-
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
@@ -94,6 +89,21 @@ jobs:
restore-keys: |
rust-docker-${{ matrix.target }}-
+ # lance's build scripts compile their own protos and look for a protoc.
+ # Point them at the one protoc-bin-vendored ships, which seaweed-worker's
+ # own build already uses, so no job depends on a system package and every
+ # build sees the same version.
+ - name: Use the vendored protoc
+ run: |
+ cd seaweed-worker
+ cargo fetch
+ # The version from the lock, not whatever else a restored cache holds.
+ version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
+ test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
+ protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
+ test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
+ echo "PROTOC=$protoc" >> "$GITHUB_ENV"
+
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
diff --git a/.github/workflows/rust-worker-tests.yml b/.github/workflows/rust-worker-tests.yml
index f6bbade3d3..ed22441020 100644
--- a/.github/workflows/rust-worker-tests.yml
+++ b/.github/workflows/rust-worker-tests.yml
@@ -36,11 +36,6 @@ jobs:
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- # lance's build scripts compile their own protos, and unlike seaweed-volume
- # they do not vendor a protoc to do it with.
- - name: Install protoc
- run: sudo apt-get install -y protobuf-compiler
-
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
@@ -58,6 +53,21 @@ jobs:
restore-keys: |
rust-worker-${{ steps.toolchain.outputs.fingerprint }}-
+ # lance's build scripts compile their own protos and look for a protoc.
+ # Point them at the one protoc-bin-vendored ships, which seaweed-worker's
+ # own build already uses, so no job depends on a system package and every
+ # build sees the same version.
+ - name: Use the vendored protoc
+ run: |
+ cd seaweed-worker
+ cargo fetch
+ # The version from the lock, not whatever else a restored cache holds.
+ version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
+ test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
+ protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
+ test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
+ echo "PROTOC=$protoc" >> "$GITHUB_ENV"
+
# The release profile is what ships, and it is where the release and the
# container builds would otherwise discover a break for the first time.
- name: Build the plugin workers
diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml
index 9cbd5df52a..a5a8b61455 100644
--- a/.github/workflows/rust_binaries_release.yml
+++ b/.github/workflows/rust_binaries_release.yml
@@ -152,11 +152,6 @@ jobs:
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
- # lance's build scripts compile their own protos, and unlike seaweed-volume
- # they do not vendor a protoc to do it with.
- - name: Install protoc
- run: sudo apt-get install -y protobuf-compiler
-
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
@@ -168,6 +163,21 @@ jobs:
restore-keys: |
rust-worker-release-${{ matrix.target }}-
+ # lance's build scripts compile their own protos and look for a protoc.
+ # Point them at the one protoc-bin-vendored ships, which seaweed-worker's
+ # own build already uses, so no job depends on a system package and every
+ # build sees the same version.
+ - name: Use the vendored protoc
+ run: |
+ cd seaweed-worker
+ cargo fetch
+ # The version from the lock, not whatever else a restored cache holds.
+ version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
+ test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
+ protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
+ test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
+ echo "PROTOC=$protoc" >> "$GITHUB_ENV"
+
- name: Build the Rust maintenance worker
run: |
cd seaweed-worker
diff --git a/seaweed-worker/Cargo.lock b/seaweed-worker/Cargo.lock
index 4fad19e505..de890b6a14 100644
--- a/seaweed-worker/Cargo.lock
+++ b/seaweed-worker/Cargo.lock
@@ -4666,6 +4666,70 @@ dependencies = [
"prost 0.14.4",
]
+[[package]]
+name = "protoc-bin-vendored"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa"
+dependencies = [
+ "protoc-bin-vendored-linux-aarch_64",
+ "protoc-bin-vendored-linux-ppcle_64",
+ "protoc-bin-vendored-linux-s390_64",
+ "protoc-bin-vendored-linux-x86_32",
+ "protoc-bin-vendored-linux-x86_64",
+ "protoc-bin-vendored-macos-aarch_64",
+ "protoc-bin-vendored-macos-x86_64",
+ "protoc-bin-vendored-win32",
+]
+
+[[package]]
+name = "protoc-bin-vendored-linux-aarch_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c"
+
+[[package]]
+name = "protoc-bin-vendored-linux-ppcle_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c"
+
+[[package]]
+name = "protoc-bin-vendored-linux-s390_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_32"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78"
+
+[[package]]
+name = "protoc-bin-vendored-macos-aarch_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092"
+
+[[package]]
+name = "protoc-bin-vendored-macos-x86_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756"
+
+[[package]]
+name = "protoc-bin-vendored-win32"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"
+
[[package]]
name = "quick-xml"
version = "0.39.4"
@@ -5379,6 +5443,7 @@ dependencies = [
"prometheus",
"prost 0.13.5",
"prost-types 0.13.5",
+ "protoc-bin-vendored",
"tokio",
"tokio-stream",
"tonic",
diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md
index 080ddfd4c4..6dc6eb6cc4 100644
--- a/seaweed-worker/README.md
+++ b/seaweed-worker/README.md
@@ -12,6 +12,17 @@ one.
`core` knows nothing about any job. A second worker is a new crate beside
`lance` that depends on it, not a fork of the protocol.
+## Building
+
+`core` compiles `plugin.proto` with the protoc that protoc-bin-vendored ships,
+the way seaweed-volume does, so it needs no system install.
+
+The lance crates compile protos of their own, in their own build-script
+processes, which nothing our build script sets can reach. They need a protoc of
+their own: either one on PATH — `brew install protobuf`, `apt install
+protobuf-compiler` — or `PROTOC` naming one. CI points it at the vendored
+binary for the runner's platform, resolved from the version in `Cargo.lock`.
+
## Running
cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646
diff --git a/seaweed-worker/crates/core/Cargo.toml b/seaweed-worker/crates/core/Cargo.toml
index 2af01bc3e3..e1ae1a10ea 100644
--- a/seaweed-worker/crates/core/Cargo.toml
+++ b/seaweed-worker/crates/core/Cargo.toml
@@ -21,3 +21,7 @@ tracing.workspace = true
[build-dependencies]
tonic-build.workspace = true
+# Ships protoc with the build so neither CI nor a developer needs a system
+# install, and so the version is pinned rather than whatever the platform's
+# package manager happens to carry. The same crate seaweed-volume uses.
+protoc-bin-vendored = "3"
diff --git a/seaweed-worker/crates/core/build.rs b/seaweed-worker/crates/core/build.rs
index 80bd222d24..07a4ab57db 100644
--- a/seaweed-worker/crates/core/build.rs
+++ b/seaweed-worker/crates/core/build.rs
@@ -1,4 +1,12 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
+ // Use the protoc that ships with protoc-bin-vendored rather than a system
+ // one, so the build needs no package manager and always sees the same
+ // version. An explicit PROTOC still wins, for packagers supplying their own
+ // and for the lance crates, whose own build scripts read the same variable.
+ if std::env::var_os("PROTOC").is_none() {
+ std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?);
+ }
+
// Compiled straight out of the Go tree, the way seaweed-volume already reads
// filer.proto, so the contract cannot drift from a vendored copy.
tonic_build::configure()