"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
testserver: fix tests failing due to stopped servers
Nick Craig-Wood55655efabf1f
diff --git a/fstest/testserver/init.d/README.md b/fstest/testserver/init.d/README.mdindex aedc67eb4..df6bf4d46 100644--- a/fstest/testserver/init.d/README.md+++ b/fstest/testserver/init.d/README.md@@ -1,11 +1,12 @@This directory contains scripts to start and stop servers for testing.-The commands are named after the remotes in use. They should be-executable files with the following parameters:+The commands are named after the remotes in use. They are executable+files with the following parameters:- start - starts the server- stop - stops the server+ start - starts the server if not running+ stop - stops the server if nothing is using itstatus - returns non-zero exit code if the server is not running+ reset - stops the server and resets any reference countsThese will be called automatically by test_all if that remote isrequired.@@ -21,16 +22,22 @@ after the connection succeeds rclone will wait `5s` before continuing.This is for servers that aren't quite ready even though they haveopened their TCP ports.-`run.bash` contains boilerplate to be included in a bash script for-interpreting the command line parameters.+## Writing new scripts-`docker.bash` contains library functions to help with docker-implementations.+A docker based server or an `rclone serve` based server should be easy+to write. Look at once of the examples.-## TODO+`run.bash` contains boilerplate to be included in a bash script for+interpreting the command line parameters. This does reference counting+to ensure multiple copies of the server aren't running at once.+Including this is mandatory. It will call your `start()`, `stop()` and+`status()` functions.-- sftpd - https://github.com/panubo/docker-sshd ?-- openstack swift - https://github.com/bouncestorage/docker-swift-- ceph - https://github.com/ceph/cn-- other ftp servers+`docker.bash` contains library functions to help with docker+implementations. It contains implementations of `stop()` and+`status()` so all you have to do is write a `start()` function.+`rclone-serve.bash` contains functions to help with `rclone serve`+based implementations. It contains implementations of `stop()` and+`status()` so all you have to do is write a `start()` function which+should call the `run()` function provided.diff --git a/fstest/testserver/init.d/run.bash b/fstest/testserver/init.d/run.bashindex ee2a14eef..478cb70c5 100644--- a/fstest/testserver/init.d/run.bash+++ b/fstest/testserver/init.d/run.bash@@ -1,17 +1,101 @@#!/usr/bin/env bash+set -euo pipefail-case "$1" in- start)- start- ;;- stop)- stop- ;;- status)- status- ;;- *)- echo "usage: $0 start|stop|status" >&2- exit 1- ;;+BASE="${STATE_DIR:-${XDG_RUNTIME_DIR:-/tmp}/rclone-test-server}"+NAME="$(basename "$0")"+ROOT="${BASE}/${NAME}"+STATE="${ROOT}/state"+LOCKF="${ROOT}/lock"+REFC="${STATE}/refcount"+ENVF="${STATE}/env"++mkdir -p "${STATE}"+[[ -f "${REFC}" ]] || echo 0 >"${REFC}"+[[ -f "${ENVF}" ]] || : >"${ENVF}"+: > "${LOCKF}" # ensure file exists++# status helper that won't trip set -e+_is_running() { set +e; status >/dev/null 2>&1; local rc=$?; set -e; return $rc; }++_acquire_lock() {+ # open fd 9 on lock file and take exclusive lock+ exec 9>"${LOCKF}"+ flock -x 9+}++_release_lock() {+ flock -u 9+ exec 9>&-+}++case "${1:-}" in+ start)+ _acquire_lock+ trap '_release_lock' EXIT++ rc=$(cat "${REFC}" 2>/dev/null || echo 0)++ if (( rc == 0 )); then+ # First client: ensure a clean instance, then start and cache env+ if _is_running; then+ stop || true+ fi+ if ! out="$(start)"; then+ echo "failed to start" >&2+ exit 1+ fi+ printf "%s\n" "$out" > "${ENVF}"+ else+ # Already owned: make sure it’s still up; if not, restart and refresh env+ if ! _is_running; then+ if ! out="$(start)"; then+ echo "failed to restart" >&2+ exit 1+ fi+ printf "%s\n" "$out" > "${ENVF}"+ fi+ fi++ rc=$((rc+1)); echo "${rc}" > "${REFC}"+ cat "${ENVF}"++ trap - EXIT+ _release_lock+ ;;++ stop)+ _acquire_lock+ trap '_release_lock' EXIT++ rc=$(cat "${REFC}" 2>/dev/null || echo 0)+ if (( rc > 0 )); then rc=$((rc-1)); fi+ echo "${rc}" > "${REFC}"+ if (( rc == 0 )) && _is_running; then+ stop || true+ fi++ trap - EXIT+ _release_lock+ ;;++ reset)+ _acquire_lock+ trap '_release_lock' EXIT++ stop || true+ rm -rf "${BASE}"++ trap - EXIT+ _release_lock+ ;;++ status)+ # passthrough; do NOT take the lock+ status+ ;;++ *)+ echo "usage: $0 {start|stop|reset|status}" >&2+ exit 2+ ;;esacdiff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.goindex 45b107243..9967bf546 100644--- a/fstest/testserver/testserver.go+++ b/fstest/testserver/testserver.go@@ -19,12 +19,8 @@ import ()var (- once sync.Once- configDir string // where the config is stored- // Note of running servers- runningMu sync.Mutex- running = map[string]int{}- errNotFound = errors.New("command not found")+ findConfigOnce sync.Once+ configDir string // where the config is stored)// Assume we are run somewhere within the rclone root@@ -42,27 +38,28 @@ func findConfig() (string, error) {return "", errors.New("couldn't find testserver config files - run from within rclone source")}+// returns path to a script to start this server+func cmdPath(name string) string {+ return filepath.Join(configDir, name)+}++// return true if the server with name has a start command+func hasStartCommand(name string) bool {+ fi, err := os.Stat(cmdPath(name))+ return err == nil && !fi.IsDir()+}+// run the command returning the output and an errorfunc run(name, command string) (out []byte, err error) {- cmdPath := filepath.Join(configDir, name)- fi, err := os.Stat(cmdPath)- if err != nil || fi.IsDir() {- return nil, errNotFound- }- cmd := exec.Command(cmdPath, command)+ script := cmdPath(name)+ cmd := exec.Command(script, command)out, err = cmd.CombinedOutput()if err != nil {- err = fmt.Errorf("failed to run %s %s\n%s: %w", cmdPath, command, string(out), err)+ err = fmt.Errorf("failed to run %s %s\n%s: %w", script, command, string(out), err)}return out, err}-// Check to see if the server is running-func isRunning(name string) bool {- _, err := run(name, "status")- return err == nil-}-// envKey returns the environment variable name to set name, keyfunc envKey(name, key string) string {return fmt.Sprintf("RCLONE_CONFIG_%s_%s", strings.ToUpper(name), strings.ToUpper(key))@@ -71,8 +68,7 @@ func envKey(name, key string) string {// match a line of config var=valuevar matchLine = regexp.MustCompile(`^([a-zA-Z_]+)=(.*)$`)-// Start the server and set its env vars-// Call with the mutex held+// Start the server and env vars so rclone can use itfunc start(name string) error {fs.Logf(name, "Starting server")out, err := run(name, "start")@@ -144,82 +140,59 @@ func start(name string) error {return fmt.Errorf("failed to connect to %q on %q", name, connect)}-// Start starts the named test server which can be stopped by the-// function returned.-func Start(remoteName string) (fn func(), err error) {- if remoteName == "" {- // don't start the local backend- return func() {}, nil+// Stops the named test server+func stop(name string) {+ fs.Logf(name, "Stopping server")+ _, err := run(name, "stop")+ if err != nil {+ fs.Errorf(name, "Failed to stop server: %v", err)+ }+}++// No server to stop so do nothing+func stopNothing() {+}++// Start starts the test server for remoteName.+//+// This must be stopped by calling the function returned when finished.+func Start(remote string) (fn func(), err error) {+ // don't start the local backend+ if remote == "" {+ return stopNothing, nil}- parsed, err := fspath.Parse(remoteName)+ parsed, err := fspath.Parse(remote)if err != nil {return nil, err}name := parsed.ConfigString+ // don't start the local backendif name == "" {- // don't start the local backend- return func() {}, nil+ return stopNothing, nil}// Make sure we know where the config is- once.Do(func() {+ findConfigOnce.Do(func() {configDir, err = findConfig()})if err != nil {return nil, err}- runningMu.Lock()- defer runningMu.Unlock()-- if running[name] <= 0 {- // if server isn't running check to see if this server has- // been started already but not by us and stop it if so- const maxTries = 10- for i := 1; i <= maxTries; i++ {- if !isRunning(name) {- fs.Logf(name, "Stopped server")- break- }- if i != 1 {- time.Sleep(time.Second)- fs.Logf(name, "Attempting to stop %s try %d/%d", name, i, maxTries)- }- stop(name)- }- if !isRunning(name) {- err = start(name)- if err == errNotFound {- // if no file found then don't start or stop- return func() {}, nil- } else if err != nil {- return nil, err- }- running[name] = 0- } else {- running[name] = 1- }+ // If remote has no start command then do nothing+ if !hasStartCommand(name) {+ return stopNothing, nil+ }++ // Start the server+ err = start(name)+ if err != nil {+ return nil, err}- running[name]+++ // And return a function to stop itreturn func() {- runningMu.Lock()- defer runningMu.Unlock()stop(name)}, nil}--// Stops the named test server-// Call with the mutex held-func stop(name string) {- running[name]--- if running[name] <= 0 {- _, err := run(name, "stop")- if err != nil {- fs.Errorf(name, "Failed to stop server: %v", err)- }- running[name] = 0- fs.Logf(name, "Stopping server")- }-}