SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
Perf CI: benchmark the Rust volume server and report memory usage (#10111)
Chris Lu2c2df751f5d7
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.ymlindex 8d492d9ca3..610ddb39e7 100644--- a/.github/workflows/performance.yml+++ b/.github/workflows/performance.yml@@ -7,6 +7,8 @@ on:- '**/*.go'- 'go.mod'- 'go.sum'+ - 'seaweed-volume/**'+ - 'test/perf/**'- '.github/workflows/performance.yml'workflow_dispatch:inputs:@@ -48,6 +50,9 @@ concurrency:permissions:contents: read+env:+ VOL_SIZE_LIMIT: "1024"+jobs:performance-profile:name: CPU and Heap Profile@@ -83,6 +88,11 @@ jobs:# give the volume server a moment to register with the mastersleep 3+ - name: Start memory sampler+ run: |+ bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &+ echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"+- name: Capture profiles under loadrun: |DURATION="${{ github.event.inputs.profile_duration || '30' }}"@@ -96,6 +106,22 @@ jobs:go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || truego tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true+ - name: Record memory usage+ if: always()+ run: |+ kill -TERM "${SAMPLER_PID}" 2>/dev/null || true+ sleep 2+ {+ echo "## Memory usage (peak RSS)"+ echo '```'+ if [ -f mem-profile.csv.peak ]; then+ awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak+ else+ echo "no memory samples captured"+ fi+ echo '```'+ } >> "$GITHUB_STEP_SUMMARY"+- name: Profile summaryif: always()run: |@@ -125,13 +151,21 @@ jobs:cpu-top.txtheap-top.txtgoroutine.txt+ mem-profile.csv+ mem-profile.csv.peakweed.logretention-days: 30benchmark:- name: Throughput Benchmark+ name: Throughput Benchmark (${{ matrix.impl }} volume)runs-on: ubuntu-latest- timeout-minutes: 45+ timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ impl: [go, rust]+ env:+ IMPL: ${{ matrix.impl }}steps:- name: Check out codeuses: actions/checkout@v7@@ -141,15 +175,41 @@ jobs:with:go-version-file: 'go.mod'+ - name: Install protobuf compiler+ if: matrix.impl == 'rust'+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler++ - name: Install Rust toolchain+ if: matrix.impl == 'rust'+ uses: dtolnay/rust-toolchain@stable++ - name: Cache cargo registry and target+ if: matrix.impl == 'rust'+ uses: actions/cache@v5+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ seaweed-volume/target+ key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}+ restore-keys: |+ rust-+- name: Build weedrun: go build -o weed_bin ./weed- - name: Start server+ - name: Build Rust volume server+ if: matrix.impl == 'rust'+ run: cd seaweed-volume && cargo build --release++ - name: Start master and volume serverrun: |- mkdir -p ./perfdata- ./weed_bin -v=1 server -dir=./perfdata -volume.max=0 \- -master.volumeSizeLimitMB=1024 > weed.log 2>&1 &- echo "WEED_PID=$!" >> "$GITHUB_ENV"+ mkdir -p ./perfdata/master ./perfdata/vol+ ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \+ -mdir=./perfdata/master -peers=none \+ -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \+ > master.log 2>&1 &+ echo "MASTER_PID=$!" >> "$GITHUB_ENV"for i in $(seq 1 60); doif curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; thenecho "master is ready"@@ -157,8 +217,33 @@ jobs:fisleep 1done+ if [ "${IMPL}" = "rust" ]; then+ ./seaweed-volume/target/release/weed-volume \+ --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \+ --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \+ > volume.log 2>&1 &+ else+ ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \+ -port=8080 -dir=./perfdata/vol -max=100 \+ > volume.log 2>&1 &+ fi+ echo "VOLUME_PID=$!" >> "$GITHUB_ENV"+ for i in $(seq 1 60); do+ if curl -sf http://localhost:8080/status >/dev/null 2>&1; then+ echo "volume server is ready"+ break+ fi+ sleep 1+ done+ # let the volume server register with the master via heartbeatsleep 3+ - name: Start memory sampler+ run: |+ bash test/perf/mem_sample.sh mem-benchmark.csv \+ "master=${MASTER_PID}" "volume=${VOLUME_PID}" &+ echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"+- name: Run throughput benchmarkrun: |N="${{ github.event.inputs.benchmark_files || '100000' }}"@@ -168,46 +253,75 @@ jobs:-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt- name: Run Go micro-benchmarks+ if: matrix.impl == 'go'continue-on-error: truerun: |go test -run='^$' -bench=. -benchmem -benchtime=10x \./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \2>&1 | tee go-benchmarks.txt+ - name: Record memory usage+ if: always()+ run: |+ kill -TERM "${SAMPLER_PID}" 2>/dev/null || true+ sleep 2+ {+ echo "## Memory usage (peak RSS, ${IMPL} volume)"+ echo '```'+ if [ -f mem-benchmark.csv.peak ]; then+ awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak+ else+ echo "no memory samples captured"+ fi+ echo '```'+ } >> "$GITHUB_STEP_SUMMARY"+- name: Benchmark summaryif: always()run: |{- echo "## Throughput benchmark"+ echo "## Throughput benchmark (${IMPL} volume)"echo '```'grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"echo '```'} >> "$GITHUB_STEP_SUMMARY"- - name: Stop server+ - name: Stop processesif: always()- run: kill "${WEED_PID}" 2>/dev/null || true+ run: |+ kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true- - name: Show server log on failure+ - name: Show logs on failureif: failure()- run: tail -200 weed.log || true+ run: |+ echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true+ echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true- name: Upload benchmark resultsif: always()uses: actions/upload-artifact@v7with:- name: benchmark-results-${{ github.run_number }}+ name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}path: |benchmark-results.txtgo-benchmarks.txt- weed.log+ mem-benchmark.csv+ mem-benchmark.csv.peak+ master.log+ volume.logretention-days: 7s3-benchmark:- name: S3 Read/Write Benchmark+ name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)runs-on: ubuntu-latest- timeout-minutes: 45+ timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ impl: [go, rust]+ env:+ IMPL: ${{ matrix.impl }}steps:- name: Check out codeuses: actions/checkout@v7@@ -217,19 +331,43 @@ jobs:with:go-version-file: 'go.mod'+ - name: Install protobuf compiler+ if: matrix.impl == 'rust'+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler++ - name: Install Rust toolchain+ if: matrix.impl == 'rust'+ uses: dtolnay/rust-toolchain@stable++ - name: Cache cargo registry and target+ if: matrix.impl == 'rust'+ uses: actions/cache@v5+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ seaweed-volume/target+ key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}+ restore-keys: |+ rust-+- name: Build weed and S3 load toolrun: |go build -o weed_bin ./weedgo build -o s3bench ./test/s3/benchmark- - name: Start server with S3 gateway+ - name: Build Rust volume server+ if: matrix.impl == 'rust'+ run: cd seaweed-volume && cargo build --release++ - name: Start cluster with S3 gatewayrun: |- mkdir -p ./perfdata- ./weed_bin -v=1 server -dir=./perfdata -s3 -filer \- -volume.max=0 -master.volumeSizeLimitMB=1024 \- -s3.port=8000 -s3.config=./docker/compose/s3.json \- > weed.log 2>&1 &- echo "WEED_PID=$!" >> "$GITHUB_ENV"+ mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer+ ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \+ -mdir=./perfdata/master -peers=none \+ -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \+ > master.log 2>&1 &+ echo "MASTER_PID=$!" >> "$GITHUB_ENV"for i in $(seq 1 60); doif curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; thenecho "master is ready"@@ -237,6 +375,29 @@ jobs:fisleep 1done+ if [ "${IMPL}" = "rust" ]; then+ ./seaweed-volume/target/release/weed-volume \+ --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \+ --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \+ > volume.log 2>&1 &+ else+ ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \+ -port=8080 -dir=./perfdata/vol -max=100 \+ > volume.log 2>&1 &+ fi+ echo "VOLUME_PID=$!" >> "$GITHUB_ENV"+ for i in $(seq 1 60); do+ if curl -sf http://localhost:8080/status >/dev/null 2>&1; then+ echo "volume server is ready"+ break+ fi+ sleep 1+ done+ sleep 3+ ./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \+ -s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \+ > filer.log 2>&1 &+ echo "FILER_PID=$!" >> "$GITHUB_ENV"for i in $(seq 1 30); doif nc -z localhost 8000 2>/dev/null; thenecho "s3 gateway is ready"@@ -246,6 +407,12 @@ jobs:donesleep 2+ - name: Start memory sampler+ run: |+ bash test/perf/mem_sample.sh mem-s3.csv \+ "master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &+ echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"+- name: Run S3 read/write benchmarkrun: |OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"@@ -256,31 +423,55 @@ jobs:-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \2>&1 | tee s3-benchmark-results.txt+ - name: Record memory usage+ if: always()+ run: |+ kill -TERM "${SAMPLER_PID}" 2>/dev/null || true+ sleep 2+ {+ echo "## Memory usage (peak RSS, ${IMPL} volume)"+ echo '```'+ if [ -f mem-s3.csv.peak ]; then+ awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak+ else+ echo "no memory samples captured"+ fi+ echo '```'+ } >> "$GITHUB_STEP_SUMMARY"+- name: S3 benchmark summaryif: always()run: |{- echo "## S3 read/write benchmark"+ echo "## S3 read/write benchmark (${IMPL} volume)"echo '```'grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"echo '```'} >> "$GITHUB_STEP_SUMMARY"- - name: Stop server+ - name: Stop processesif: always()- run: kill "${WEED_PID}" 2>/dev/null || true+ run: |+ kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true- - name: Show server log on failure+ - name: Show logs on failureif: failure()- run: tail -200 weed.log || true+ run: |+ echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true+ echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true+ echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true- name: Upload S3 benchmark resultsif: always()uses: actions/upload-artifact@v7with:- name: s3-benchmark-results-${{ github.run_number }}+ name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}path: |s3-benchmark-results.txt- weed.log+ mem-s3.csv+ mem-s3.csv.peak+ master.log+ volume.log+ filer.logretention-days: 7diff --git a/test/perf/mem_sample.sh b/test/perf/mem_sample.shnew file mode 100755index 0000000000..503f2adce6--- /dev/null+++ b/test/perf/mem_sample.sh@@ -0,0 +1,62 @@+#!/usr/bin/env bash+# Sample resident memory of one or more processes once per second.+# Usage: mem_sample.sh <out.csv> <label=pid> [label=pid ...]+#+# Writes a per-second time series to <out.csv> and, on stop, the peak+# resident set (VmHWM) per process to <out.csv>.peak. Stop with SIGTERM/SIGINT;+# if a process already exited, its peak falls back to the max RSS sampled.+# Linux only (reads /proc/<pid>/status).+set -u++if [ "$#" -lt 2 ]; then+ echo "Usage: $0 <out.csv> <label=pid> [label=pid ...]" >&2+ exit 1+fi++out="$1"; shift++labels=(); pids=(); peaks=()+for arg in "$@"; do+ labels+=("${arg%%=*}")+ pids+=("${arg#*=}")+ peaks+=(0)+done++record_peaks() {+ local total=0 i hwm+ : > "${out}.peak"+ for i in "${!pids[@]}"; do+ hwm="$(awk '/^VmHWM:/{print $2}' "/proc/${pids[$i]}/status" 2>/dev/null)"+ if [ -z "$hwm" ] || [ "$hwm" -eq 0 ]; then+ hwm="${peaks[$i]}"+ fi+ printf '%s\t%s\n' "${labels[$i]}" "$hwm" >> "${out}.peak"+ total=$((total + hwm))+ done+ printf 'total\t%s\n' "$total" >> "${out}.peak"+ exit 0+}+trap record_peaks TERM INT++header="epoch"+for l in "${labels[@]}"; do header+=",${l}_rss_kb"; done+header+=",total_rss_kb"+echo "$header" > "$out"++while true; do+ line="$(date +%s)"; total=0+ for i in "${!pids[@]}"; do+ p="${pids[$i]}"+ rss="$(awk '/^VmRSS:/{print $2}' "/proc/$p/status" 2>/dev/null)"+ rss="${rss:-0}"+ if [ "$rss" -gt "${peaks[$i]}" ]; then+ peaks[$i]="$rss"+ fi+ line+=",${rss}"+ total=$((total + rss))+ done+ echo "${line},${total}" >> "$out"+ # Sleep via a background child + wait so a caught signal interrupts promptly.+ sleep 1 &+ wait $! 2>/dev/null+done