Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

Perf CI: benchmark the Rust volume server and report memory usage (#10111)

Chris Lu2c2df751f5d7
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
index 8d492d9ca3..610ddb39e7 100644
--- a/.github/workflows/performance.yml
+++ b/.github/workflows/performance.yml
@@ -7,6 +7,8 @@ on:
- '**/*.go'
- 'go.mod'
- 'go.sum'
+ - 'seaweed-volume/**'
+ - 'test/perf/**'
- '.github/workflows/performance.yml'
workflow_dispatch:
inputs:
@@ -48,6 +50,9 @@ concurrency:
permissions:
contents: read
+env:
+ VOL_SIZE_LIMIT: "1024"
+
jobs:
performance-profile:
name: CPU and Heap Profile
@@ -83,6 +88,11 @@ jobs:
# give the volume server a moment to register with the master
sleep 3
+ - name: Start memory sampler
+ run: |
+ bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
+ echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
+
- name: Capture profiles under load
run: |
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
@@ -96,6 +106,22 @@ jobs:
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
+ - name: Record memory usage
+ if: always()
+ run: |
+ kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
+ sleep 2
+ {
+ echo "## Memory usage (peak RSS)"
+ echo '```'
+ if [ -f mem-profile.csv.peak ]; then
+ awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
+ else
+ echo "no memory samples captured"
+ fi
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+
- name: Profile summary
if: always()
run: |
@@ -125,13 +151,21 @@ jobs:
cpu-top.txt
heap-top.txt
goroutine.txt
+ mem-profile.csv
+ mem-profile.csv.peak
weed.log
retention-days: 30
benchmark:
- name: Throughput Benchmark
+ name: Throughput Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
- timeout-minutes: 45
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ impl: [go, rust]
+ env:
+ IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
@@ -141,15 +175,41 @@ jobs:
with:
go-version-file: 'go.mod'
+ - name: Install protobuf compiler
+ if: matrix.impl == 'rust'
+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+ - name: Install Rust toolchain
+ if: matrix.impl == 'rust'
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache cargo registry and target
+ if: matrix.impl == 'rust'
+ uses: actions/cache@v5
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ seaweed-volume/target
+ key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+ restore-keys: |
+ rust-
+
- name: Build weed
run: go build -o weed_bin ./weed
- - name: Start server
+ - name: Build Rust volume server
+ if: matrix.impl == 'rust'
+ run: cd seaweed-volume && cargo build --release
+
+ - name: Start master and volume server
run: |
- mkdir -p ./perfdata
- ./weed_bin -v=1 server -dir=./perfdata -volume.max=0 \
- -master.volumeSizeLimitMB=1024 > weed.log 2>&1 &
- echo "WEED_PID=$!" >> "$GITHUB_ENV"
+ mkdir -p ./perfdata/master ./perfdata/vol
+ ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
+ -mdir=./perfdata/master -peers=none \
+ -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
+ > master.log 2>&1 &
+ echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
@@ -157,8 +217,33 @@ jobs:
fi
sleep 1
done
+ if [ "${IMPL}" = "rust" ]; then
+ ./seaweed-volume/target/release/weed-volume \
+ --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
+ --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
+ > volume.log 2>&1 &
+ else
+ ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
+ -port=8080 -dir=./perfdata/vol -max=100 \
+ > volume.log 2>&1 &
+ fi
+ echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
+ for i in $(seq 1 60); do
+ if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
+ echo "volume server is ready"
+ break
+ fi
+ sleep 1
+ done
+ # let the volume server register with the master via heartbeat
sleep 3
+ - name: Start memory sampler
+ run: |
+ bash test/perf/mem_sample.sh mem-benchmark.csv \
+ "master=${MASTER_PID}" "volume=${VOLUME_PID}" &
+ echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
+
- name: Run throughput benchmark
run: |
N="${{ github.event.inputs.benchmark_files || '100000' }}"
@@ -168,46 +253,75 @@ jobs:
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
- name: Run Go micro-benchmarks
+ if: matrix.impl == 'go'
continue-on-error: true
run: |
go test -run='^$' -bench=. -benchmem -benchtime=10x \
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
2>&1 | tee go-benchmarks.txt
+ - name: Record memory usage
+ if: always()
+ run: |
+ kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
+ sleep 2
+ {
+ echo "## Memory usage (peak RSS, ${IMPL} volume)"
+ echo '```'
+ if [ -f mem-benchmark.csv.peak ]; then
+ awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
+ else
+ echo "no memory samples captured"
+ fi
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+
- name: Benchmark summary
if: always()
run: |
{
- echo "## Throughput benchmark"
+ echo "## Throughput benchmark (${IMPL} volume)"
echo '```'
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- - name: Stop server
+ - name: Stop processes
if: always()
- run: kill "${WEED_PID}" 2>/dev/null || true
+ run: |
+ kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- - name: Show server log on failure
+ - name: Show logs on failure
if: failure()
- run: tail -200 weed.log || true
+ run: |
+ echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
+ echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
- name: Upload benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
- name: benchmark-results-${{ github.run_number }}
+ name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
benchmark-results.txt
go-benchmarks.txt
- weed.log
+ mem-benchmark.csv
+ mem-benchmark.csv.peak
+ master.log
+ volume.log
retention-days: 7
s3-benchmark:
- name: S3 Read/Write Benchmark
+ name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
- timeout-minutes: 45
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ impl: [go, rust]
+ env:
+ IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
@@ -217,19 +331,43 @@ jobs:
with:
go-version-file: 'go.mod'
+ - name: Install protobuf compiler
+ if: matrix.impl == 'rust'
+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+ - name: Install Rust toolchain
+ if: matrix.impl == 'rust'
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache cargo registry and target
+ if: matrix.impl == 'rust'
+ uses: actions/cache@v5
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ seaweed-volume/target
+ key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+ restore-keys: |
+ rust-
+
- name: Build weed and S3 load tool
run: |
go build -o weed_bin ./weed
go build -o s3bench ./test/s3/benchmark
- - name: Start server with S3 gateway
+ - name: Build Rust volume server
+ if: matrix.impl == 'rust'
+ run: cd seaweed-volume && cargo build --release
+
+ - name: Start cluster with S3 gateway
run: |
- mkdir -p ./perfdata
- ./weed_bin -v=1 server -dir=./perfdata -s3 -filer \
- -volume.max=0 -master.volumeSizeLimitMB=1024 \
- -s3.port=8000 -s3.config=./docker/compose/s3.json \
- > weed.log 2>&1 &
- echo "WEED_PID=$!" >> "$GITHUB_ENV"
+ mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
+ ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
+ -mdir=./perfdata/master -peers=none \
+ -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
+ > master.log 2>&1 &
+ echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
@@ -237,6 +375,29 @@ jobs:
fi
sleep 1
done
+ if [ "${IMPL}" = "rust" ]; then
+ ./seaweed-volume/target/release/weed-volume \
+ --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
+ --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
+ > volume.log 2>&1 &
+ else
+ ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
+ -port=8080 -dir=./perfdata/vol -max=100 \
+ > volume.log 2>&1 &
+ fi
+ echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
+ for i in $(seq 1 60); do
+ if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
+ echo "volume server is ready"
+ break
+ fi
+ sleep 1
+ done
+ sleep 3
+ ./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
+ -s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
+ > filer.log 2>&1 &
+ echo "FILER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 30); do
if nc -z localhost 8000 2>/dev/null; then
echo "s3 gateway is ready"
@@ -246,6 +407,12 @@ jobs:
done
sleep 2
+ - name: Start memory sampler
+ run: |
+ bash test/perf/mem_sample.sh mem-s3.csv \
+ "master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
+ echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
+
- name: Run S3 read/write benchmark
run: |
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
@@ -256,31 +423,55 @@ jobs:
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
2>&1 | tee s3-benchmark-results.txt
+ - name: Record memory usage
+ if: always()
+ run: |
+ kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
+ sleep 2
+ {
+ echo "## Memory usage (peak RSS, ${IMPL} volume)"
+ echo '```'
+ if [ -f mem-s3.csv.peak ]; then
+ awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
+ else
+ echo "no memory samples captured"
+ fi
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+
- name: S3 benchmark summary
if: always()
run: |
{
- echo "## S3 read/write benchmark"
+ echo "## S3 read/write benchmark (${IMPL} volume)"
echo '```'
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- - name: Stop server
+ - name: Stop processes
if: always()
- run: kill "${WEED_PID}" 2>/dev/null || true
+ run: |
+ kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- - name: Show server log on failure
+ - name: Show logs on failure
if: failure()
- run: tail -200 weed.log || true
+ run: |
+ echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
+ echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
+ echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
- name: Upload S3 benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
- name: s3-benchmark-results-${{ github.run_number }}
+ name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
s3-benchmark-results.txt
- weed.log
+ mem-s3.csv
+ mem-s3.csv.peak
+ master.log
+ volume.log
+ filer.log
retention-days: 7
diff --git a/test/perf/mem_sample.sh b/test/perf/mem_sample.sh
new file mode 100755
index 0000000000..503f2adce6
--- /dev/null
+++ b/test/perf/mem_sample.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Sample resident memory of one or more processes once per second.
+# Usage: mem_sample.sh <out.csv> <label=pid> [label=pid ...]
+#
+# Writes a per-second time series to <out.csv> and, on stop, the peak
+# resident set (VmHWM) per process to <out.csv>.peak. Stop with SIGTERM/SIGINT;
+# if a process already exited, its peak falls back to the max RSS sampled.
+# Linux only (reads /proc/<pid>/status).
+set -u
+
+if [ "$#" -lt 2 ]; then
+ echo "Usage: $0 <out.csv> <label=pid> [label=pid ...]" >&2
+ exit 1
+fi
+
+out="$1"; shift
+
+labels=(); pids=(); peaks=()
+for arg in "$@"; do
+ labels+=("${arg%%=*}")
+ pids+=("${arg#*=}")
+ peaks+=(0)
+done
+
+record_peaks() {
+ local total=0 i hwm
+ : > "${out}.peak"
+ for i in "${!pids[@]}"; do
+ hwm="$(awk '/^VmHWM:/{print $2}' "/proc/${pids[$i]}/status" 2>/dev/null)"
+ if [ -z "$hwm" ] || [ "$hwm" -eq 0 ]; then
+ hwm="${peaks[$i]}"
+ fi
+ printf '%s\t%s\n' "${labels[$i]}" "$hwm" >> "${out}.peak"
+ total=$((total + hwm))
+ done
+ printf 'total\t%s\n' "$total" >> "${out}.peak"
+ exit 0
+}
+trap record_peaks TERM INT
+
+header="epoch"
+for l in "${labels[@]}"; do header+=",${l}_rss_kb"; done
+header+=",total_rss_kb"
+echo "$header" > "$out"
+
+while true; do
+ line="$(date +%s)"; total=0
+ for i in "${!pids[@]}"; do
+ p="${pids[$i]}"
+ rss="$(awk '/^VmRSS:/{print $2}' "/proc/$p/status" 2>/dev/null)"
+ rss="${rss:-0}"
+ if [ "$rss" -gt "${peaks[$i]}" ]; then
+ peaks[$i]="$rss"
+ fi
+ line+=",${rss}"
+ total=$((total + rss))
+ done
+ echo "${line},${total}" >> "$out"
+ # Sleep via a background child + wait so a caught signal interrupts promptly.
+ sleep 1 &
+ wait $! 2>/dev/null
+done