SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
#!/usr/bin/env bash
#
# Locking / concurrency test battery for Samba on a SeaweedFS FUSE mount.
#
# Covers the challenges a network-filesystem backend has to get right:
# 1. POSIX fcntl byte-range locking on the FUSE mount (SetLk/GetLk)
# 2. Distributed locking (-dlm): a write held open on one mount blocks a
# writer on another mount until it is released
# 3. Distributed locking integrity: concurrent writers to the same file from
# two mounts produce intact (non-torn) data
# 4. Concurrent writers to distinct files all succeed
#
# Required env:
# SMB_USER, SMB_PASS samba credentials
# MOUNT_SHARE dir on the smbd-backed FUSE mount (mount 1)
# MOUNT2_SHARE dir on the second FUSE mount (mount 2)
# Optional env:
# SMB_HOST (127.0.0.1), SMB_SHARE (seaweedfs), SMB_PORT (445)
set -uo pipefail
SMB_HOST="${SMB_HOST:-127.0.0.1}"
SMB_SHARE="${SMB_SHARE:-seaweedfs}"
SMB_PORT="${SMB_PORT:-445}"
SMB_USER="${SMB_USER:?SMB_USER is required}"
SMB_PASS="${SMB_PASS:?SMB_PASS is required}"
MOUNT_SHARE="${MOUNT_SHARE:?MOUNT_SHARE is required}"
MOUNT2_SHARE="${MOUNT2_SHARE:?MOUNT2_SHARE is required}"
WORK="$(mktemp -d /tmp/samba-locktest.XXXXXX)"
trap 'rm -rf "${WORK}"' EXIT
PASS=0
FAIL=0
pass() { printf ' [PASS] %s\n' "$1"; PASS=$((PASS + 1)); }
fail() { printf ' [FAIL] %s\n' "$1"; FAIL=$((FAIL + 1)); }
smb() {
smbclient "//${SMB_HOST}/${SMB_SHARE}" -p "${SMB_PORT}" \
-U "${SMB_USER}%${SMB_PASS}" -m SMB3 -c "$1"
}
md5() { md5sum "$1" | awk '{print $1}'; }
# 1. POSIX fcntl byte-range locking on the FUSE mount ------------------------
# Exercises the mount's SetLk/GetLk via two processes contending over fcntl
# (F_SETLK) byte-range locks. python3's fcntl.lockf issues real POSIX locks.
echo "==> 1. POSIX fcntl byte-range locking (FUSE mount SetLk/GetLk)"
lockfile="${MOUNT_SHARE}/fcntl_lock.dat"
: >"${lockfile}"
fcntl_out="$(python3 - "${lockfile}" <<'PY'
import fcntl, os, sys
path = sys.argv[1]
parent_to_child_r, parent_to_child_w = os.pipe() # release signal
child_to_parent_r, child_to_parent_w = os.pipe() # locked signal
pid = os.fork()
if pid == 0: # child: hold an exclusive lock on [0,100)
fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
fcntl.lockf(fd, fcntl.LOCK_EX, 100, 0, 0)
os.write(child_to_parent_w, b"L")
os.read(parent_to_child_r, 1) # wait until parent says release
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
os.close(fd)
os._exit(0)
# parent
os.read(child_to_parent_r, 1) # wait until child holds the lock
fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
results = []
# a. a conflicting exclusive lock must be denied while the child holds it
try:
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 0, 0)
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
results.append(("conflicting exclusive lock denied while held", False))
except OSError:
results.append(("conflicting exclusive lock denied while held", True))
# b. a non-overlapping range must be grantable
try:
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 200, 0)
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 200, 0)
results.append(("non-overlapping range lock granted", True))
except OSError:
results.append(("non-overlapping range lock granted", False))
# c. after the holder releases, the lock must be acquirable
os.write(parent_to_child_w, b"R")
os.waitpid(pid, 0)
try:
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 0, 0)
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
results.append(("lock acquirable after holder releases", True))
except OSError:
results.append(("lock acquirable after holder releases", False))
for name, ok in results:
print((" [PASS] " if ok else " [FAIL] ") + name)
sys.exit(0 if all(ok for _, ok in results) else 1)
PY
)"
echo "${fcntl_out}"
PASS=$((PASS + $(grep -c '\[PASS\]' <<<"${fcntl_out}")))
FAIL=$((FAIL + $(grep -c '\[FAIL\]' <<<"${fcntl_out}")))
# 2. Distributed lock blocks a cross-mount writer, then hands it off ----------
# mount 2 holds a file open for writing (holding the DLM lock on its path).
# An SMB put of the same file goes through mount 1 and must (a) block while
# mount 2 holds it and (b) succeed once mount 2 releases, leaving the SMB
# writer's payload on disk. smbclient gets a long client timeout (-t) so we are
# testing the lock handoff itself, not smbclient's own ~20s default timeout.
echo "==> 2. distributed lock: cross-mount write coordination"
dlmfile="dlm_coord.bin"
newdata="${WORK}/dlm_new.bin"
head -c 4096 /dev/urandom >"${newdata}"
# Hold the file open for writing on mount 2 via fd 9 -> holds the DLM lock.
exec 9>"${MOUNT2_SHARE}/${dlmfile}"
printf 'held-by-mount2' >&9
# Start the SMB write; record its real exit code when it returns. The subshell
# must NOT inherit fd 9 (9>&-): otherwise the SMB writer keeps the file open and
# waits on a DLM lock held by its own inherited descriptor, deadlocking the
# handoff this test is meant to exercise.
rm -f "${WORK}/dlm_put.rc"
(
smbclient "//${SMB_HOST}/${SMB_SHARE}" -p "${SMB_PORT}" \
-U "${SMB_USER}%${SMB_PASS}" -m SMB3 -t 120 \
-c "put ${newdata} ${dlmfile}" >/dev/null 2>&1
echo "$?" >"${WORK}/dlm_put.rc"
) 9>&- &
smb_bg=$!
sleep 4
if [[ ! -f "${WORK}/dlm_put.rc" ]]; then
pass "SMB write blocks while another mount holds the file open"
else
fail "SMB write returned early instead of blocking (rc=$(cat "${WORK}/dlm_put.rc"))"
fi
# Release mount 2's DLM lock; the blocked SMB write must now complete.
exec 9>&-
# Wait (bounded) for the SMB put to finish so a stuck handoff fails the test
# instead of hanging the suite.
put_rc="timeout"
for _ in $(seq 1 20); do
if [[ -f "${WORK}/dlm_put.rc" ]]; then
put_rc="$(cat "${WORK}/dlm_put.rc")"
break
fi
sleep 1
done
kill "${smb_bg}" 2>/dev/null
wait "${smb_bg}" 2>/dev/null
if [[ "${put_rc}" == "0" ]]; then
pass "blocked SMB write succeeds after the other mount releases"
else
fail "blocked SMB write succeeds after the other mount releases (rc=${put_rc})"
fi
# A correct handoff leaves the SMB writer's payload on disk: mount 1 acquired
# the lock and wrote after mount 2 released.
got="${WORK}/dlm_got.bin"
if smb "get ${dlmfile} ${got}" >/dev/null 2>&1 && [[ "$(md5 "${got}")" == "$(md5 "${newdata}")" ]]; then
pass "post-release content is the SMB writer's payload (correct handoff)"
else
fail "post-release content is the SMB writer's payload (correct handoff)"
fi
# 3. Distributed lock integrity: concurrent writers, same file ---------------
# An SMB writer (mount 1) and a direct writer (mount 2) race on one file. DLM
# serializes them, so the result must be exactly one of the two payloads.
echo "==> 3. distributed lock: concurrent writers produce intact data"
racefile="dlm_race.bin"
payloadA="${WORK}/dlm_raceA.bin"
head -c 1048576 /dev/urandom >"${payloadA}"
payloadB="direct-write-from-mount2-payload"
(smb "put ${payloadA} ${racefile}" >/dev/null 2>&1) &
(printf '%s' "${payloadB}" >"${MOUNT2_SHARE}/${racefile}") &
wait
racegot="${WORK}/dlm_race_got.bin"
if smb "get ${racefile} ${racegot}" >/dev/null 2>&1 &&
{ [[ "$(md5 "${racegot}")" == "$(md5 "${payloadA}")" ]] || [[ "$(cat "${racegot}")" == "${payloadB}" ]]; }; then
pass "concurrent same-file writers leave one intact payload"
else
fail "concurrent same-file writers leave one intact payload"
fi
# 4. Concurrent writers to distinct files ------------------------------------
echo "==> 4. concurrent writers to distinct files"
n=6
declare -a srcs=()
for i in $(seq 1 "${n}"); do
s="${WORK}/cc_${i}.bin"
head -c 1048576 /dev/urandom >"${s}"
srcs+=("${s}")
(smb "put ${s} concurrent_${i}.bin" >/dev/null 2>&1) &
done
wait
all_ok=true
for i in $(seq 1 "${n}"); do
g="${WORK}/cc_got_${i}.bin"
if ! smb "get concurrent_${i}.bin ${g}" >/dev/null 2>&1 ||
[[ "$(md5 "${srcs[$((i - 1))]}")" != "$(md5 "${g}")" ]]; then
all_ok=false
fi
done
if ${all_ok}; then
pass "${n} concurrent distinct-file writes all intact"
else
fail "${n} concurrent distinct-file writes all intact"
fi
echo
echo "==> Summary: ${PASS} passed, ${FAIL} failed"
[[ "${FAIL}" -eq 0 ]]