Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

8.8 KB
Raw Blame
//go:build linux || darwin

package fuse_dlm

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same
// file concurrently produce valid (non-corrupted) data. With DLM enabled, the
// writes are serialized — one blocks until the other completes.
//
// Note: cross-mount read consistency depends on FUSE kernel cache invalidation
// and filer metadata subscription, which are asynchronous. This test verifies
// write integrity, not instant read convergence.
func TestDLMConcurrentWritersSameFile(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping DLM integration test in short mode")
	}

	cluster := startDLMTestCluster(t)
	t.Cleanup(cluster.Stop)

	const iterations = 5
	for iter := 0; iter < iterations; iter++ {
		fileName := fmt.Sprintf("concurrent_write_%d.txt", iter)
		payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter))
		payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter))

		var wg sync.WaitGroup
		wg.Add(2)

		go func() {
			defer wg.Done()
			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644)
			assert.NoError(t, err, "mount0 write iteration %d", iter)
		}()

		go func() {
			defer wg.Done()
			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644)
			assert.NoError(t, err, "mount1 write iteration %d", iter)
		}()

		wg.Wait()

		// Verify file is readable and contains one of the expected payloads
		// (read from mount0 — its own view is authoritative for write success).
		content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
		require.NoError(t, err, "read from mount0 iteration %d", iter)
		validPayload := string(content) == string(payloadA) || string(content) == string(payloadB)
		assert.True(t, validPayload,
			"iteration %d: content must be one of the expected payloads, got: %q", iter, content)
	}
}

// TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles
// from both mounts all succeed without errors.
func TestDLMRepeatedOpenWriteClose(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping DLM integration test in short mode")
	}

	cluster := startDLMTestCluster(t)
	t.Cleanup(cluster.Stop)

	const cycles = 20
	fileName := "repeated_write.txt"

	var wg sync.WaitGroup
	wg.Add(2)

	go func() {
		defer wg.Done()
		for i := 0; i < cycles; i++ {
			data := []byte(fmt.Sprintf("mount0-cycle-%d", i))
			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644)
			assert.NoError(t, err, "mount0 cycle %d", i)
		}
	}()

	go func() {
		defer wg.Done()
		for i := 0; i < cycles; i++ {
			data := []byte(fmt.Sprintf("mount1-cycle-%d", i))
			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644)
			assert.NoError(t, err, "mount1 cycle %d", i)
		}
	}()

	wg.Wait()

	// File must be readable from at least one mount
	content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
	require.NoError(t, err)
	assert.NotEmpty(t, content, "file must not be empty")
}

// TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one
// mount has a file open for writing, another mount's write-open blocks until
// the first mount closes the file.
func TestDLMWriteBlocksSecondWriter(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping DLM integration test in short mode")
	}

	cluster := startDLMTestCluster(t)
	t.Cleanup(cluster.Stop)

	fileName := "blocking_test.txt"
	path0 := filepath.Join(cluster.mountPoints[0], fileName)
	path1 := filepath.Join(cluster.mountPoints[1], fileName)

	// Mount 0 opens the file for writing and holds it open
	f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	require.NoError(t, err, "mount0 open")
	_, err = f.Write([]byte("mount0-holds-lock"))
	require.NoError(t, err, "mount0 write")

	// Mount 1 tries to write — should block (we use a goroutine with atomic flag)
	var mount1Completed atomic.Bool
	mount1Done := make(chan error, 1)
	go func() {
		err := os.WriteFile(path1, []byte("mount1-waited"), 0644)
		mount1Completed.Store(true)
		mount1Done <- err
	}()

	// Give mount 1 a moment — it should NOT complete while mount 0 holds the file open
	time.Sleep(3 * time.Second)
	require.False(t, mount1Completed.Load(),
		"mount1 write must not complete while mount0 holds the file open")
	t.Log("mount1 write is blocked as expected while mount0 holds the file")

	// Mount 0 closes the file — this releases the DLM lock
	require.NoError(t, f.Close(), "mount0 close")

	// Mount 1 should now complete
	select {
	case err := <-mount1Done:
		assert.NoError(t, err, "mount1 write after mount0 close")
	case <-time.After(30 * time.Second):
		t.Fatal("mount1 write did not complete within 30s after mount0 closed")
	}
}

// TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM:
// while mount0 has a file open for writing (re-opened after creation),
// mount1 cannot rename it until mount0 closes the file.
func TestDLMRenameWhileWriteOpen(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping DLM integration test in short mode")
	}

	cluster := startDLMTestCluster(t)
	t.Cleanup(cluster.Stop)

	origName := "rename_source.txt"
	newName := "rename_dest.txt"

	// Create and close the file first so it's flushed to the filer and
	// visible on both mounts.
	require.NoError(t, os.WriteFile(
		filepath.Join(cluster.mountPoints[0], origName),
		[]byte("initial-content"), 0644))
	time.Sleep(2 * time.Second) // metadata propagation

	// Verify mount1 can see the file
	_, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName))
	require.NoError(t, err, "mount1 should see the file")

	// Mount 0 re-opens the file for writing and holds it open
	f, err := os.OpenFile(
		filepath.Join(cluster.mountPoints[0], origName),
		os.O_WRONLY|os.O_TRUNC, 0644)
	require.NoError(t, err, "mount0 reopen")
	_, err = f.Write([]byte("data-while-holding-lock"))
	require.NoError(t, err, "mount0 write")

	// Mount 1 tries to rename — should block because mount0 holds the
	// DLM lock on the old path
	var renameCompleted atomic.Bool
	renameDone := make(chan error, 1)
	go func() {
		err := os.Rename(
			filepath.Join(cluster.mountPoints[1], origName),
			filepath.Join(cluster.mountPoints[1], newName))
		renameCompleted.Store(true)
		renameDone <- err
	}()

	// Rename must NOT complete while mount0 holds the file open
	time.Sleep(3 * time.Second)
	require.False(t, renameCompleted.Load(),
		"rename must not complete while mount0 holds the file open")
	t.Log("rename is blocked as expected while mount0 holds the file")

	// Mount 0 closes → releases DLM lock → rename should proceed
	require.NoError(t, f.Close(), "mount0 close")

	select {
	case err := <-renameDone:
		assert.NoError(t, err, "rename after mount0 close")
	case <-time.After(30 * time.Second):
		t.Fatal("rename did not complete within 30s after mount0 closed")
	}
}

// Note: Same-mount rename while a file is open for writing is not tested here
// because macOS FUSE serializes operations on the same inode, causing a
// kernel-level deadlock between the Rename handler's internal flush and the
// pending Close. Same-mount coordination is already handled by the per-mount
// fhLockTable and FUSE kernel serialization, so DLM is not needed for it.

// TestDLMConcurrentRenames verifies that two concurrent renames of the same
// file from different mounts don't corrupt metadata. DLM locks on both old
// and new paths ensure renames are serialized.
func TestDLMConcurrentRenames(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping DLM integration test in short mode")
	}

	cluster := startDLMTestCluster(t)
	t.Cleanup(cluster.Stop)

	// Create a file first
	origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt")
	require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644))
	time.Sleep(1 * time.Second) // propagation

	// Both mounts try to rename the same file concurrently
	var wg sync.WaitGroup
	var errA, errB error
	wg.Add(2)

	go func() {
		defer wg.Done()
		errA = os.Rename(
			filepath.Join(cluster.mountPoints[0], "rename_race.txt"),
			filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"),
		)
	}()

	go func() {
		defer wg.Done()
		errB = os.Rename(
			filepath.Join(cluster.mountPoints[1], "rename_race.txt"),
			filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"),
		)
	}()

	wg.Wait()

	// At least one rename should succeed; the other may fail with ENOENT
	// since the source was already moved.
	succeeded := 0
	if errA == nil {
		succeeded++
		t.Logf("mount0 rename succeeded")
	} else {
		t.Logf("mount0 rename failed: %v", errA)
	}
	if errB == nil {
		succeeded++
		t.Logf("mount1 rename succeeded")
	} else {
		t.Logf("mount1 rename failed: %v", errB)
	}
	assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed")
}