"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
build: make integration tests run better on macOS and Windows
Nick Craig-Wood8dc4c0120909
diff --git a/fstest/testserver/init.d/PORTS.md b/fstest/testserver/init.d/PORTS.mdnew file mode 100644index 000000000..f8e75c4e2--- /dev/null+++ b/fstest/testserver/init.d/PORTS.md@@ -0,0 +1,41 @@+# Ports for tests++All these tests need to run on a different port.++They should be bound to localhost so they are not accessible externally.++| Port | Test |+|:-----:|:----:|+| 88 | TestHdfs |+| 750 | TestHdfs |+| 8020 | TestHdfs |+| 8086 | TestSeafileV6 |+| 8087 | TestSeafile |+| 8088 | TestSeafileEncrypted |+| 9866 | TestHdfs |+| 28620 | TestWebdavRclone |+| 28621 | TestSFTPRclone |+| 28622 | TestFTPRclone |+| 28623 | TestSFTPRcloneSSH |+| 28624 | TestS3Rclone |+| 28625 | TestS3Minio |+| 28626 | TestS3MinioEdge |+| 28627 | TestSFTPOpenssh |+| 28628 | TestSwiftAIO |+| 28629 | TestWebdavNextcloud |+| 28630 | TestSMB |+| 28631 | TestFTPProftpd |+| 38081 | TestWebdavOwncloud |++## Non localhost tests++All these use `$(docker_ip)` which means they don't work on macOS or+Windows. It is proabably possible to make them work with some effort+but will require port forwarding a range of ports and configuring the+FTP server to only use that range of ports. The FTP server will likely+need know it is behind a NAT so it advertises the correct external IP.++- TestFTPProftpd+- TestFTPPureftpd+- TestFTPVsftpd+- TestFTPVsftpdTLSdiff --git a/fstest/testserver/init.d/TestS3Minio b/fstest/testserver/init.d/TestS3Minioindex 6dc41af5b..4a599d457 100755--- a/fstest/testserver/init.d/TestS3Minio+++ b/fstest/testserver/init.d/TestS3Minio@@ -5,6 +5,7 @@ set -eNAME=minioUSER=rclonePASS=AxedBodedGinger7+PORT=28625. $(dirname "$0")/docker.bash@@ -12,14 +13,15 @@ start() {docker run --rm -d --name $NAME \-e "MINIO_ACCESS_KEY=$USER" \-e "MINIO_SECRET_KEY=$PASS" \+ -p 127.0.0.1:${PORT}:9000 \minio/minio server /dataecho type=s3echo provider=Minioecho access_key_id=$USERecho secret_access_key=$PASS- echo endpoint=http://$(docker_ip):9000/- echo _connect=$(docker_ip):9000+ echo endpoint=http://127.0.0.1:${PORT}/+ echo _connect=127.0.0.1:${PORT}}. $(dirname "$0")/run.bashdiff --git a/fstest/testserver/init.d/TestS3MinioEdge b/fstest/testserver/init.d/TestS3MinioEdgeindex f48002920..459dbb903 100755--- a/fstest/testserver/init.d/TestS3MinioEdge+++ b/fstest/testserver/init.d/TestS3MinioEdge@@ -5,6 +5,7 @@ set -eNAME=minio-edgeUSER=rclonePASS=DeniseOxygenEiffel4+PORT=28626. $(dirname "$0")/docker.bash@@ -12,14 +13,15 @@ start() {docker run --rm -d --name $NAME \-e "MINIO_ACCESS_KEY=$USER" \-e "MINIO_SECRET_KEY=$PASS" \+ -p 127.0.0.1:${PORT}:9000 \minio/minio:edge server /dataecho type=s3echo provider=Minioecho access_key_id=$USERecho secret_access_key=$PASS- echo endpoint=http://$(docker_ip):9000/- echo _connect=$(docker_ip):9000+ echo endpoint=http://127.0.0.1:${PORT}/+ echo _connect=127.0.0.1:${PORT}}. $(dirname "$0")/run.bashdiff --git a/fstest/testserver/init.d/TestSFTPOpenssh b/fstest/testserver/init.d/TestSFTPOpensshindex 4eb14ac9a..5b67f7923 100755--- a/fstest/testserver/init.d/TestSFTPOpenssh+++ b/fstest/testserver/init.d/TestSFTPOpenssh@@ -5,19 +5,22 @@ set -eNAME=rclone-sftp-opensshUSER=rclonePASS=password+PORT=28627. $(dirname "$0")/docker.bashstart() {docker run --rm -d --name ${NAME} \+ -p 127.0.0.1:${PORT}:22 \rclone/test-sftp-opensshecho type=sftp- echo host=$(docker_ip)+ echo host=127.0.0.1+ echo port=$PORTecho user=$USERecho pass=$(rclone obscure $PASS)echo copy_is_hardlink=true- echo _connect=$(docker_ip):22+ echo _connect=127.0.0.1:${PORT}}. $(dirname "$0")/run.bashdiff --git a/fstest/testserver/init.d/TestSMB b/fstest/testserver/init.d/TestSMBindex 788b5a84a..56bf16552 100755--- a/fstest/testserver/init.d/TestSMB+++ b/fstest/testserver/init.d/TestSMB@@ -6,11 +6,15 @@ NAME=smbUSER=rclonePASS=GNF3CqeuWORKGROUP=thepub+PORT=28630. $(dirname "$0")/docker.bashstart() {- docker run --rm -d --name $NAME dperson/samba \+ docker run --rm -d --name $NAME \+ -p 127.0.0.1:${PORT}:445 \+ -p 127.0.0.1:${PORT}:445/udp \+ dperson/samba \-p \-u "rclone;${PASS}" \-w "${WORKGROUP}" \@@ -18,11 +22,12 @@ start() {-s "rclone;/rclone;yes;no;no;rclone"echo type=smb- echo host=$(docker_ip)+ echo host=127.0.0.1echo user=$USER+ echo port=$PORTecho pass=$(rclone obscure $PASS)echo domain=$WORKGROUP- echo _connect=$(docker_ip):139+ echo _connect=127.0.0.1:${PORT}}. $(dirname "$0")/run.bashdiff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIOindex 5192f2055..1f5ca8c70 100755--- a/fstest/testserver/init.d/TestSwiftAIO+++ b/fstest/testserver/init.d/TestSwiftAIO@@ -3,19 +3,21 @@set -eNAME=swift-aio+PORT=28628. $(dirname "$0")/docker.bashstart() {docker run --rm -d --name ${NAME} \+ -p 127.0.0.1:${PORT}:8080 \bouncestorage/swift-aioecho type=swiftecho env_auth=falseecho user=test:testerecho key=testing- echo auth=http://$(docker_ip):8080/auth/v1.0- echo _connect=$(docker_ip):8080+ echo auth=http://127.0.0.1:${PORT}/auth/v1.0+ echo _connect=127.0.0.1:${PORT}}. $(dirname "$0")/run.bashdiff --git a/fstest/testserver/init.d/TestWebdavNextcloud b/fstest/testserver/init.d/TestWebdavNextcloudindex 9cf391e38..1fe8c6e86 100755--- a/fstest/testserver/init.d/TestWebdavNextcloud+++ b/fstest/testserver/init.d/TestWebdavNextcloud@@ -5,6 +5,7 @@ set -eNAME=nextcloudUSER=rclonePASS=ArmorAbleMale6+PORT=28629. $(dirname "$0")/docker.bash@@ -14,14 +15,15 @@ start() {-e "NEXTCLOUD_ADMIN_USER=rclone" \-e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \-e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \+ -p 127.0.0.1:${PORT}:80 \nextcloud:latestecho type=webdav- echo url=http://$(docker_ip)/remote.php/dav/files/$USER/+ echo url=http://127.0.0.1:${PORT}/remote.php/dav/files/$USER/echo user=$USERecho pass=$(rclone obscure $PASS)echo vendor=nextcloud- echo _connect=$(docker_ip):80+ echo _connect=127.0.0.1:${PORT}}. $(dirname "$0")/run.bashdiff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.goindex 005096e85..9cc1354d2 100644--- a/fstest/testserver/testserver.go+++ b/fstest/testserver/testserver.go@@ -103,14 +103,31 @@ func start(name string) error {}// If we got a _connect value then try to connect to itconst maxTries = 30+ var rdBuf = make([]byte, 1)for i := 1; i <= maxTries; i++ {+ if i != 0 {+ time.Sleep(time.Second)+ }fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)- conn, err := net.Dial("tcp", connect)- if err == nil {- _ = conn.Close()- return nil+ conn, err := net.DialTimeout("tcp", connect, time.Second)+ if err != nil {+ fs.Debugf(name, "Connection to %q failed try %d/%d: %v", connect, i, maxTries, err)+ continue}- time.Sleep(time.Second)++ err = conn.SetReadDeadline(time.Now().Add(time.Second))+ if err != nil {+ return fmt.Errorf("failed to set deadline: %w", err)+ }+ n, err := conn.Read(rdBuf)+ _ = conn.Close()+ fs.Debugf(name, "Read %d, error: %v", n, err)+ if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) {+ // Try again+ continue+ }+ //time.Sleep(30 * time.Second)+ return nil}return fmt.Errorf("failed to connect to %q on %q", name, connect)}