SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
ci: build protoc into the crate instead of installing it per job (#10830)
Chris Lubaead6901c58
diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.ymlindex 8d72fa4a7c..11b339bc6d 100644--- a/.github/workflows/container_dev.yml+++ b/.github/workflows/container_dev.yml@@ -31,9 +31,6 @@ jobs:- name: Checkoutuses: actions/checkout@v7- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablewith:diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.ymlindex 23193e573f..a82755c2eb 100644--- a/.github/workflows/container_latest.yml+++ b/.github/workflows/container_latest.yml@@ -76,9 +76,6 @@ jobs:with:ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablewith:diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.ymlindex 198de19ef8..f8d9cd21d1 100644--- a/.github/workflows/container_release_unified.yml+++ b/.github/workflows/container_release_unified.yml@@ -59,9 +59,6 @@ jobs:- name: Checkoutuses: actions/checkout@v7- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablewith:diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.ymlindex cd5c521da3..2be3782bd7 100644--- a/.github/workflows/performance.yml+++ b/.github/workflows/performance.yml@@ -175,10 +175,6 @@ jobs:with:go-version-file: 'go.mod'- - name: Install protobuf compiler- if: matrix.impl == 'rust'- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainif: matrix.impl == 'rust'uses: dtolnay/rust-toolchain@stable@@ -331,10 +327,6 @@ jobs:with:go-version-file: 'go.mod'- - name: Install protobuf compiler- if: matrix.impl == 'rust'- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainif: matrix.impl == 'rust'uses: dtolnay/rust-toolchain@stablediff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.ymlindex 6f2417296b..ba8e09a395 100644--- a/.github/workflows/rust-volume-server-tests.yml+++ b/.github/workflows/rust-volume-server-tests.yml@@ -36,9 +36,6 @@ jobs:- name: Checkout codeuses: actions/checkout@v7- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stable@@ -79,9 +76,6 @@ jobs:with:go-version-file: 'go.mod'- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stable@@ -164,9 +158,6 @@ jobs:with:go-version-file: 'go.mod'- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablediff --git a/.github/workflows/rust_binaries_dev.yml b/.github/workflows/rust_binaries_dev.ymlindex 9c4aacc716..c8cb4f0c68 100644--- a/.github/workflows/rust_binaries_dev.yml+++ b/.github/workflows/rust_binaries_dev.yml@@ -41,9 +41,6 @@ jobs:steps:- uses: actions/checkout@v7- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stable@@ -110,9 +107,6 @@ jobs:steps:- uses: actions/checkout@v7- - name: Install protobuf compiler- run: brew install protobuf-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablewith:diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.ymlindex 46c596fc30..14a49e8353 100644--- a/.github/workflows/rust_binaries_release.yml+++ b/.github/workflows/rust_binaries_release.yml@@ -28,9 +28,6 @@ jobs:steps:- uses: actions/checkout@v7- - name: Install protobuf compiler- run: sudo apt-get update && sudo apt-get install -y protobuf-compiler-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablewith:@@ -131,9 +128,6 @@ jobs:steps:- uses: actions/checkout@v7- - name: Install protobuf compiler- run: brew install protobuf-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablewith:@@ -211,9 +205,6 @@ jobs:steps:- uses: actions/checkout@v7- - name: Install protobuf compiler- run: choco install protoc -y-- name: Install Rust toolchainuses: dtolnay/rust-toolchain@stablediff --git a/seaweed-volume/Cargo.lock b/seaweed-volume/Cargo.lockindex a4c8752585..b0ce04703a 100644--- a/seaweed-volume/Cargo.lock+++ b/seaweed-volume/Cargo.lock@@ -2929,6 +2929,70 @@ dependencies = ["prost 0.13.5",]+[[package]]+name = "protoc-bin-vendored"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa"+dependencies = [+ "protoc-bin-vendored-linux-aarch_64",+ "protoc-bin-vendored-linux-ppcle_64",+ "protoc-bin-vendored-linux-s390_64",+ "protoc-bin-vendored-linux-x86_32",+ "protoc-bin-vendored-linux-x86_64",+ "protoc-bin-vendored-macos-aarch_64",+ "protoc-bin-vendored-macos-x86_64",+ "protoc-bin-vendored-win32",+]++[[package]]+name = "protoc-bin-vendored-linux-aarch_64"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c"++[[package]]+name = "protoc-bin-vendored-linux-ppcle_64"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c"++[[package]]+name = "protoc-bin-vendored-linux-s390_64"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0"++[[package]]+name = "protoc-bin-vendored-linux-x86_32"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5"++[[package]]+name = "protoc-bin-vendored-linux-x86_64"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78"++[[package]]+name = "protoc-bin-vendored-macos-aarch_64"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092"++[[package]]+name = "protoc-bin-vendored-macos-x86_64"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756"++[[package]]+name = "protoc-bin-vendored-win32"+version = "3.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"+[[package]]name = "pxfm"version = "0.1.28"@@ -4529,6 +4593,7 @@ dependencies = ["prometheus","prost 0.13.5","prost-types 0.13.5",+ "protoc-bin-vendored","rand 0.10.2","redb","reed-solomon-erasure",diff --git a/seaweed-volume/Cargo.toml b/seaweed-volume/Cargo.tomlindex 481f7bc12f..91db873ea4 100644--- a/seaweed-volume/Cargo.toml+++ b/seaweed-volume/Cargo.toml@@ -137,6 +137,10 @@ tempfile = "3"[build-dependencies]tonic-build = "0.12"+# Ships protoc with the build so neither CI nor a developer needs a system+# install, and so the version is pinned rather than whatever the platform's+# package manager happens to carry.+protoc-bin-vendored = "3"[patch.crates-io]reed-solomon-erasure = { path = "vendor/reed-solomon-erasure" }diff --git a/seaweed-volume/build.rs b/seaweed-volume/build.rsindex a42b8bdfc9..99af370a35 100644--- a/seaweed-volume/build.rs+++ b/seaweed-volume/build.rs@@ -1,10 +1,18 @@fn main() -> Result<(), Box<dyn std::error::Error>> {+ // Use the protoc that ships with protoc-bin-vendored rather than a system+ // one, so the build needs no package manager and always sees the same+ // version. An explicit PROTOC still wins, for packagers supplying their own.+ if std::env::var_os("PROTOC").is_none() {+ std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?);+ }+let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?);tonic_build::configure().build_server(true).build_client(true)- // filer.proto uses proto3 optional; older protoc (e.g. 3.12 from ubuntu-22.04 apt)- // rejects it without this flag, and newer protoc still accepts the flag+ // filer.proto uses proto3 optional, which protoc rejects without this+ // flag before 3.15. The vendored protoc is newer, but it still accepts+ // the flag, so this keeps a build against an older PROTOC working..protoc_arg("--experimental_allow_proto3_optional").file_descriptor_set_path(out_dir.join("seaweed_descriptor.bin")).compile_protos(