Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

ci: build protoc into the crate instead of installing it per job (#10830)

Chris Lubaead6901c58
diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml
index 8d72fa4a7c..11b339bc6d 100644
--- a/.github/workflows/container_dev.yml
+++ b/.github/workflows/container_dev.yml
@@ -31,9 +31,6 @@ jobs:
- name: Checkout
uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml
index 23193e573f..a82755c2eb 100644
--- a/.github/workflows/container_latest.yml
+++ b/.github/workflows/container_latest.yml
@@ -76,9 +76,6 @@ jobs:
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml
index 198de19ef8..f8d9cd21d1 100644
--- a/.github/workflows/container_release_unified.yml
+++ b/.github/workflows/container_release_unified.yml
@@ -59,9 +59,6 @@ jobs:
- name: Checkout
uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index cd5c521da3..2be3782bd7 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -175,10 +175,6 @@ jobs:
with:
go-version-file: 'go.mod'
- - name: Install protobuf compiler
- if: matrix.impl == 'rust'
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
@@ -331,10 +327,6 @@ jobs:
with:
go-version-file: 'go.mod'
- - name: Install protobuf compiler
- if: matrix.impl == 'rust'
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml
index 6f2417296b..ba8e09a395 100644
--- a/.github/workflows/rust-volume-server-tests.yml
+++ b/.github/workflows/rust-volume-server-tests.yml
@@ -36,9 +36,6 @@ jobs:
- name: Checkout code
uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
@@ -79,9 +76,6 @@ jobs:
with:
go-version-file: 'go.mod'
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
@@ -164,9 +158,6 @@ jobs:
with:
go-version-file: 'go.mod'
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
diff --git a/.github/workflows/rust_binaries_dev.yml b/.github/workflows/rust_binaries_dev.yml
index 9c4aacc716..c8cb4f0c68 100644
--- a/.github/workflows/rust_binaries_dev.yml
+++ b/.github/workflows/rust_binaries_dev.yml
@@ -41,9 +41,6 @@ jobs:
steps:
- uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
@@ -110,9 +107,6 @@ jobs:
steps:
- uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: brew install protobuf
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml
index 46c596fc30..14a49e8353 100644
--- a/.github/workflows/rust_binaries_release.yml
+++ b/.github/workflows/rust_binaries_release.yml
@@ -28,9 +28,6 @@ jobs:
steps:
- uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
@@ -131,9 +128,6 @@ jobs:
steps:
- uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: brew install protobuf
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
@@ -211,9 +205,6 @@ jobs:
steps:
- uses: actions/checkout@v7
- - name: Install protobuf compiler
- run: choco install protoc -y
-
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
diff --git a/seaweed-volume/Cargo.lock b/seaweed-volume/Cargo.lock
index a4c8752585..b0ce04703a 100644
--- a/seaweed-volume/Cargo.lock
+++ b/seaweed-volume/Cargo.lock
@@ -2929,6 +2929,70 @@ dependencies = [
"prost 0.13.5",
]
+[[package]]
+name = "protoc-bin-vendored"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa"
+dependencies = [
+ "protoc-bin-vendored-linux-aarch_64",
+ "protoc-bin-vendored-linux-ppcle_64",
+ "protoc-bin-vendored-linux-s390_64",
+ "protoc-bin-vendored-linux-x86_32",
+ "protoc-bin-vendored-linux-x86_64",
+ "protoc-bin-vendored-macos-aarch_64",
+ "protoc-bin-vendored-macos-x86_64",
+ "protoc-bin-vendored-win32",
+]
+
+[[package]]
+name = "protoc-bin-vendored-linux-aarch_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c"
+
+[[package]]
+name = "protoc-bin-vendored-linux-ppcle_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c"
+
+[[package]]
+name = "protoc-bin-vendored-linux-s390_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_32"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78"
+
+[[package]]
+name = "protoc-bin-vendored-macos-aarch_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092"
+
+[[package]]
+name = "protoc-bin-vendored-macos-x86_64"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756"
+
+[[package]]
+name = "protoc-bin-vendored-win32"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"
+
[[package]]
name = "pxfm"
version = "0.1.28"
@@ -4529,6 +4593,7 @@ dependencies = [
"prometheus",
"prost 0.13.5",
"prost-types 0.13.5",
+ "protoc-bin-vendored",
"rand 0.10.2",
"redb",
"reed-solomon-erasure",
diff --git a/seaweed-volume/Cargo.toml b/seaweed-volume/Cargo.toml
index 481f7bc12f..91db873ea4 100644
--- a/seaweed-volume/Cargo.toml
+++ b/seaweed-volume/Cargo.toml
@@ -137,6 +137,10 @@ tempfile = "3"
[build-dependencies]
tonic-build = "0.12"
+# Ships protoc with the build so neither CI nor a developer needs a system
+# install, and so the version is pinned rather than whatever the platform's
+# package manager happens to carry.
+protoc-bin-vendored = "3"
[patch.crates-io]
reed-solomon-erasure = { path = "vendor/reed-solomon-erasure" }
diff --git a/seaweed-volume/build.rs b/seaweed-volume/build.rs
index a42b8bdfc9..99af370a35 100644
--- a/seaweed-volume/build.rs
+++ b/seaweed-volume/build.rs
@@ -1,10 +1,18 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
+ // Use the protoc that ships with protoc-bin-vendored rather than a system
+ // one, so the build needs no package manager and always sees the same
+ // version. An explicit PROTOC still wins, for packagers supplying their own.
+ if std::env::var_os("PROTOC").is_none() {
+ std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?);
+ }
+
let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?);
tonic_build::configure()
.build_server(true)
.build_client(true)
- // filer.proto uses proto3 optional; older protoc (e.g. 3.12 from ubuntu-22.04 apt)
- // rejects it without this flag, and newer protoc still accepts the flag
+ // filer.proto uses proto3 optional, which protoc rejects without this
+ // flag before 3.15. The vendored protoc is newer, but it still accepts
+ // the flag, so this keeps a build against an older PROTOC working.
.protoc_arg("--experimental_allow_proto3_optional")
.file_descriptor_set_path(out_dir.join("seaweed_descriptor.bin"))
.compile_protos(