SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package mount
import (
"context"
"errors"
"fmt"
"syscall"
"testing"
"time"
"github.com/seaweedfs/go-fuse/v2/fuse"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// Regression: wfs_save.go used to wrap the gRPC error with fmt.Errorf(... %v ...)
// which stringified the status and made status.FromError fall through to the
// default EIO mapping. Wrapping with %w must preserve the code so that
// codes.Canceled from a closing filer connection surfaces as ETIMEDOUT (a
// retryable hint for FUSE callers) rather than EIO.
func TestGrpcErrorToFuseStatusUnwrapsCanceledThroughFmtErrorf(t *testing.T) {
grpcErr := status.Error(codes.Canceled, "grpc: the client connection is closing")
wrapped := fmt.Errorf("UpdateEntry dir /some/path: %w", grpcErr)
got := grpcErrorToFuseStatus(wrapped)
want := fuse.Status(syscall.ETIMEDOUT)
if got != want {
t.Fatalf("grpcErrorToFuseStatus(canceled wrapped with %%w) = %v, want %v", got, want)
}
}
// Guard against regressing the wrap verb: %v loses the gRPC status and the
// classifier must fall through to EIO. This test documents that behavior so
// anyone reverting the %w change sees the intent.
func TestGrpcErrorToFuseStatusDropsCanceledThroughPercentV(t *testing.T) {
grpcErr := status.Error(codes.Canceled, "grpc: the client connection is closing")
wrapped := fmt.Errorf("UpdateEntry dir /some/path: %v", grpcErr)
got := grpcErrorToFuseStatus(wrapped)
if got != fuse.EIO {
t.Fatalf("grpcErrorToFuseStatus(canceled wrapped with %%v) = %v, want EIO (regression guard)", got)
}
}
// A full cluster must surface as ENOSPC so cp/rsync abort with "No space left
// on device" instead of a generic I/O error. The master's message reaches the
// mount stringified inside AssignVolumeResponse.Error, in one of two shapes:
// the load-shedding gRPC status text or the topology's plain error text.
func TestWriteErrorToFuseStatus(t *testing.T) {
cases := []struct {
name string
err error
want fuse.Status
}{
{
"assign shed during growth",
errors.New("upload data: filerGrpcAddress assign volume: assign volume failure count:1: assign volume: rpc error: code = ResourceExhausted desc = no writable volumes for replication:000 collection:, volume growth in progress"),
fuse.Status(syscall.ENOSPC),
},
{
"no free volumes left",
errors.New("flush data: upload data: assign volume: No writable volumes available for collection: replication:000 ttl: and no free volumes left for {replication:000}"),
fuse.Status(syscall.ENOSPC),
},
{
"unrelated upload failure",
errors.New("upload data: put http://volume:8080/3,0144: connection reset by peer"),
fuse.EIO,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := writeErrorToFuseStatus(tc.err); got != tc.want {
t.Fatalf("writeErrorToFuseStatus(%v) = %v, want %v", tc.err, got, tc.want)
}
})
}
}
func TestIsRetryableFilerError(t *testing.T) {
cases := []struct {
name string
err error
want bool
}{
{"nil", nil, false},
{"canceled", status.Error(codes.Canceled, "grpc: the client connection is closing"), true},
{"unavailable", status.Error(codes.Unavailable, "connection refused"), true},
{"deadline_exceeded", status.Error(codes.DeadlineExceeded, "deadline exceeded"), true},
{"resource_exhausted", status.Error(codes.ResourceExhausted, "too many concurrent requests"), true},
{"internal", status.Error(codes.Internal, "server crashed"), true},
{"not_found", status.Error(codes.NotFound, "entry missing"), false},
{"already_exists", status.Error(codes.AlreadyExists, "duplicate"), false},
{"invalid_argument", status.Error(codes.InvalidArgument, "bad request"), false},
{"permission_denied", status.Error(codes.PermissionDenied, "no access"), false},
{"unauthenticated", status.Error(codes.Unauthenticated, "missing creds"), false},
{"failed_precondition", status.Error(codes.FailedPrecondition, "not empty"), false},
{"create_condition_failed", fmt.Errorf("fh flush create /f: CreateEntry /f: %w", filer_pb.ErrPreconditionFailed), false},
{"plain_error_retries", errors.New("random network glitch"), true},
{"wrapped_canceled_still_retries", fmt.Errorf("ctx: %w", status.Error(codes.Canceled, "closing")), true},
{"wrapped_not_found_still_skipped", fmt.Errorf("ctx: %w", status.Error(codes.NotFound, "gone")), false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := isRetryableFilerError(tc.err); got != tc.want {
t.Fatalf("isRetryableFilerError(%v) = %v, want %v", tc.err, got, tc.want)
}
})
}
}
// retryMetadataFlushIf must short-circuit on non-retryable errors so that
// synchronous FUSE ops (chmod/utimes/xattr) don't hang for ~7s on ENOENT/
// EACCES/EINVAL.
func TestRetryMetadataFlushIfShortCircuitsOnPermanentError(t *testing.T) {
originalSleep := metadataFlushSleep
t.Cleanup(func() {
metadataFlushSleep = originalSleep
})
metadataFlushSleep = func(_ context.Context, _ time.Duration) {
t.Fatal("sleep should not be called when shouldRetry returns false")
}
attempts := 0
permanent := status.Error(codes.NotFound, "entry missing")
err := retryMetadataFlushIf(context.Background(), func() error {
attempts++
return permanent
}, isRetryableFilerError, nil)
if attempts != 1 {
t.Fatalf("attempts = %d, want 1 (permanent error should short-circuit)", attempts)
}
if !errors.Is(err, permanent) {
t.Fatalf("err = %v, want permanent sentinel", err)
}
}
// Transient errors must keep retrying up to the attempt cap even when a
// predicate is supplied.
func TestRetryMetadataFlushIfRetriesTransientErrors(t *testing.T) {
originalSleep := metadataFlushSleep
t.Cleanup(func() {
metadataFlushSleep = originalSleep
})
metadataFlushSleep = func(_ context.Context, _ time.Duration) {}
attempts := 0
transient := status.Error(codes.Canceled, "grpc: the client connection is closing")
err := retryMetadataFlushIf(context.Background(), func() error {
attempts++
return transient
}, isRetryableFilerError, nil)
if attempts != metadataFlushRetries+1 {
t.Fatalf("attempts = %d, want %d", attempts, metadataFlushRetries+1)
}
if !errors.Is(err, transient) {
t.Fatalf("err = %v, want transient sentinel", err)
}
}