Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

build: make integration tests run better on macOS and Windows

Nick Craig-Wood8dc4c0120909
diff --git a/fstest/testserver/init.d/PORTS.md b/fstest/testserver/init.d/PORTS.md
new file mode 100644
index 000000000..f8e75c4e2
--- /dev/null
+++ b/fstest/testserver/init.d/PORTS.md
@@ -0,0 +1,41 @@
+# Ports for tests
+
+All these tests need to run on a different port.
+
+They should be bound to localhost so they are not accessible externally.
+
+| Port | Test |
+|:-----:|:----:|
+| 88 | TestHdfs |
+| 750 | TestHdfs |
+| 8020 | TestHdfs |
+| 8086 | TestSeafileV6 |
+| 8087 | TestSeafile |
+| 8088 | TestSeafileEncrypted |
+| 9866 | TestHdfs |
+| 28620 | TestWebdavRclone |
+| 28621 | TestSFTPRclone |
+| 28622 | TestFTPRclone |
+| 28623 | TestSFTPRcloneSSH |
+| 28624 | TestS3Rclone |
+| 28625 | TestS3Minio |
+| 28626 | TestS3MinioEdge |
+| 28627 | TestSFTPOpenssh |
+| 28628 | TestSwiftAIO |
+| 28629 | TestWebdavNextcloud |
+| 28630 | TestSMB |
+| 28631 | TestFTPProftpd |
+| 38081 | TestWebdavOwncloud |
+
+## Non localhost tests
+
+All these use `$(docker_ip)` which means they don't work on macOS or
+Windows. It is proabably possible to make them work with some effort
+but will require port forwarding a range of ports and configuring the
+FTP server to only use that range of ports. The FTP server will likely
+need know it is behind a NAT so it advertises the correct external IP.
+
+- TestFTPProftpd
+- TestFTPPureftpd
+- TestFTPVsftpd
+- TestFTPVsftpdTLS
diff --git a/fstest/testserver/init.d/TestS3Minio b/fstest/testserver/init.d/TestS3Minio
index 6dc41af5b..4a599d457 100755
--- a/fstest/testserver/init.d/TestS3Minio
+++ b/fstest/testserver/init.d/TestS3Minio
@@ -5,6 +5,7 @@ set -e
NAME=minio
USER=rclone
PASS=AxedBodedGinger7
+PORT=28625
. $(dirname "$0")/docker.bash
@@ -12,14 +13,15 @@ start() {
docker run --rm -d --name $NAME \
-e "MINIO_ACCESS_KEY=$USER" \
-e "MINIO_SECRET_KEY=$PASS" \
+ -p 127.0.0.1:${PORT}:9000 \
minio/minio server /data
echo type=s3
echo provider=Minio
echo access_key_id=$USER
echo secret_access_key=$PASS
- echo endpoint=http://$(docker_ip):9000/
- echo _connect=$(docker_ip):9000
+ echo endpoint=http://127.0.0.1:${PORT}/
+ echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestS3MinioEdge b/fstest/testserver/init.d/TestS3MinioEdge
index f48002920..459dbb903 100755
--- a/fstest/testserver/init.d/TestS3MinioEdge
+++ b/fstest/testserver/init.d/TestS3MinioEdge
@@ -5,6 +5,7 @@ set -e
NAME=minio-edge
USER=rclone
PASS=DeniseOxygenEiffel4
+PORT=28626
. $(dirname "$0")/docker.bash
@@ -12,14 +13,15 @@ start() {
docker run --rm -d --name $NAME \
-e "MINIO_ACCESS_KEY=$USER" \
-e "MINIO_SECRET_KEY=$PASS" \
+ -p 127.0.0.1:${PORT}:9000 \
minio/minio:edge server /data
echo type=s3
echo provider=Minio
echo access_key_id=$USER
echo secret_access_key=$PASS
- echo endpoint=http://$(docker_ip):9000/
- echo _connect=$(docker_ip):9000
+ echo endpoint=http://127.0.0.1:${PORT}/
+ echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestSFTPOpenssh b/fstest/testserver/init.d/TestSFTPOpenssh
index 4eb14ac9a..5b67f7923 100755
--- a/fstest/testserver/init.d/TestSFTPOpenssh
+++ b/fstest/testserver/init.d/TestSFTPOpenssh
@@ -5,19 +5,22 @@ set -e
NAME=rclone-sftp-openssh
USER=rclone
PASS=password
+PORT=28627
. $(dirname "$0")/docker.bash
start() {
docker run --rm -d --name ${NAME} \
+ -p 127.0.0.1:${PORT}:22 \
rclone/test-sftp-openssh
echo type=sftp
- echo host=$(docker_ip)
+ echo host=127.0.0.1
+ echo port=$PORT
echo user=$USER
echo pass=$(rclone obscure $PASS)
echo copy_is_hardlink=true
- echo _connect=$(docker_ip):22
+ echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestSMB b/fstest/testserver/init.d/TestSMB
index 788b5a84a..56bf16552 100755
--- a/fstest/testserver/init.d/TestSMB
+++ b/fstest/testserver/init.d/TestSMB
@@ -6,11 +6,15 @@ NAME=smb
USER=rclone
PASS=GNF3Cqeu
WORKGROUP=thepub
+PORT=28630
. $(dirname "$0")/docker.bash
start() {
- docker run --rm -d --name $NAME dperson/samba \
+ docker run --rm -d --name $NAME \
+ -p 127.0.0.1:${PORT}:445 \
+ -p 127.0.0.1:${PORT}:445/udp \
+ dperson/samba \
-p \
-u "rclone;${PASS}" \
-w "${WORKGROUP}" \
@@ -18,11 +22,12 @@ start() {
-s "rclone;/rclone;yes;no;no;rclone"
echo type=smb
- echo host=$(docker_ip)
+ echo host=127.0.0.1
echo user=$USER
+ echo port=$PORT
echo pass=$(rclone obscure $PASS)
echo domain=$WORKGROUP
- echo _connect=$(docker_ip):139
+ echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIO
index 5192f2055..1f5ca8c70 100755
--- a/fstest/testserver/init.d/TestSwiftAIO
+++ b/fstest/testserver/init.d/TestSwiftAIO
@@ -3,19 +3,21 @@
set -e
NAME=swift-aio
+PORT=28628
. $(dirname "$0")/docker.bash
start() {
docker run --rm -d --name ${NAME} \
+ -p 127.0.0.1:${PORT}:8080 \
bouncestorage/swift-aio
echo type=swift
echo env_auth=false
echo user=test:tester
echo key=testing
- echo auth=http://$(docker_ip):8080/auth/v1.0
- echo _connect=$(docker_ip):8080
+ echo auth=http://127.0.0.1:${PORT}/auth/v1.0
+ echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash
diff --git a/fstest/testserver/init.d/TestWebdavNextcloud b/fstest/testserver/init.d/TestWebdavNextcloud
index 9cf391e38..1fe8c6e86 100755
--- a/fstest/testserver/init.d/TestWebdavNextcloud
+++ b/fstest/testserver/init.d/TestWebdavNextcloud
@@ -5,6 +5,7 @@ set -e
NAME=nextcloud
USER=rclone
PASS=ArmorAbleMale6
+PORT=28629
. $(dirname "$0")/docker.bash
@@ -14,14 +15,15 @@ start() {
-e "NEXTCLOUD_ADMIN_USER=rclone" \
-e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \
-e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \
+ -p 127.0.0.1:${PORT}:80 \
nextcloud:latest
echo type=webdav
- echo url=http://$(docker_ip)/remote.php/dav/files/$USER/
+ echo url=http://127.0.0.1:${PORT}/remote.php/dav/files/$USER/
echo user=$USER
echo pass=$(rclone obscure $PASS)
echo vendor=nextcloud
- echo _connect=$(docker_ip):80
+ echo _connect=127.0.0.1:${PORT}
}
. $(dirname "$0")/run.bash
diff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.go
index 005096e85..9cc1354d2 100644
--- a/fstest/testserver/testserver.go
+++ b/fstest/testserver/testserver.go
@@ -103,14 +103,31 @@ func start(name string) error {
}
// If we got a _connect value then try to connect to it
const maxTries = 30
+ var rdBuf = make([]byte, 1)
for i := 1; i <= maxTries; i++ {
+ if i != 0 {
+ time.Sleep(time.Second)
+ }
fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
- conn, err := net.Dial("tcp", connect)
- if err == nil {
- _ = conn.Close()
- return nil
+ conn, err := net.DialTimeout("tcp", connect, time.Second)
+ if err != nil {
+ fs.Debugf(name, "Connection to %q failed try %d/%d: %v", connect, i, maxTries, err)
+ continue
}
- time.Sleep(time.Second)
+
+ err = conn.SetReadDeadline(time.Now().Add(time.Second))
+ if err != nil {
+ return fmt.Errorf("failed to set deadline: %w", err)
+ }
+ n, err := conn.Read(rdBuf)
+ _ = conn.Close()
+ fs.Debugf(name, "Read %d, error: %v", n, err)
+ if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) {
+ // Try again
+ continue
+ }
+ //time.Sleep(30 * time.Second)
+ return nil
}
return fmt.Errorf("failed to connect to %q on %q", name, connect)
}