Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve docker: fix tests leaving unkillable processes and stale FUSE mounts

Nick Craig-Woode855d2ed36b6
diff --git a/cmd/serve/docker/docker_test.go b/cmd/serve/docker/docker_test.go
index ce7bb193a..9fa52223c 100644
--- a/cmd/serve/docker/docker_test.go
+++ b/cmd/serve/docker/docker_test.go
@@ -6,14 +6,17 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"net"
"net/http"
"os"
+ "os/exec"
"path/filepath"
"runtime"
"strings"
+ "syscall"
"testing"
"time"
@@ -398,6 +401,50 @@ func (a *APIClient) request(path string, in, out any, wantErr bool) {
time.Sleep(tempDelay)
}
+// writeMountedFile writes data to a file on a FUSE mount served by this
+// process.
+//
+// It avoids os.WriteFile as files opened with os.OpenFile are handed
+// to the Go runtime poller and the kernel then asks the FUSE server
+// to poll them from epoll_ctl and epoll_wait. The server lives in
+// this process and a thread waiting inside epoll cannot be preempted,
+// so a garbage collection starting while a poll is outstanding stops
+// the world for good. The poll is never answered, the process cannot
+// be killed even with SIGKILL and the mount is left behind.
+// See: https://github.com/golang/go/issues/21014
+//
+// os.NewFile only returns a pollable file for a descriptor which is
+// already in non-blocking mode, so a descriptor straight from open(2)
+// stays out of the poller.
+func writeMountedFile(path string, data []byte, perm uint32) (err error) {
+ fd, err := syscall.Open(path, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_TRUNC, perm)
+ if err != nil {
+ return err
+ }
+ f := os.NewFile(uintptr(fd), path)
+ defer fs.CheckClose(f, &err)
+ // Deadlines work on polled files only
+ if err := f.SetDeadline(time.Time{}); !errors.Is(err, os.ErrNoDeadline) {
+ return fmt.Errorf("%s is being polled by the runtime which risks deadlocking the test: %w", path, err)
+ }
+ _, err = f.Write(data)
+ return err
+}
+
+// checkUnmounted fails the test and tears the mount down if mountpoint
+// is still mounted.
+func checkUnmounted(t *testing.T, mountpoint string) {
+ // CheckMountReady and fusermount are both Linux only
+ if !mountlib.CanCheckMountReady || mountlib.CheckMountReady(mountpoint) != nil {
+ return
+ }
+ assert.Fail(t, fmt.Sprintf("mountpoint %s was left mounted", mountpoint))
+ // Lazy unmount as the mount may still be busy
+ if out, err := exec.Command("fusermount", "-uz", mountpoint).CombinedOutput(); err != nil {
+ t.Logf("Failed to unmount %s: %v: %s", mountpoint, err, out)
+ }
+}
+
func testMountAPI(t *testing.T, sockAddr string) {
// Disable tests under macOS and linux in the CI since they are locking up
if runtime.GOOS == "darwin" || runtime.GOOS == "linux" {
@@ -412,6 +459,7 @@ func testMountAPI(t *testing.T, sockAddr string) {
testDir, testFs := initialise(ctx, t)
err := config.SetCacheDir(testDir)
require.NoError(t, err)
+ mount1 := filepath.Join(testDir, "vol1")
defer func() {
_ = config.SetCacheDir(oldCacheDir)
if !t.Failed() {
@@ -419,6 +467,7 @@ func testMountAPI(t *testing.T, sockAddr string) {
_ = os.RemoveAll(testDir)
}
}()
+ defer checkUnmounted(t, mount1)
// Prepare API client
var cli *APIClient
@@ -457,7 +506,6 @@ func testMountAPI(t *testing.T, sockAddr string) {
// Run test sequence
path1 := filepath.Join(testDir, "path1")
require.NoError(t, file.MkdirAll(path1, 0755))
- mount1 := filepath.Join(testDir, "vol1")
res := ""
cli.request("Activate", "{}", &res, false)
@@ -485,7 +533,7 @@ func testMountAPI(t *testing.T, sockAddr string) {
assert.Contains(t, res, "volume is in use")
text := []byte("banana")
- err = os.WriteFile(filepath.Join(mount1, "txt"), text, 0644)
+ err = writeMountedFile(filepath.Join(mount1, "txt"), text, 0644)
assert.NoError(t, err)
time.Sleep(tempDelay)