Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3api: fix TestRoutedMultipartFinalize flake from a reused fake filer port (#12571)

Chris Lua0f95e16bc24
diff --git a/weed/s3api/s3api_object_routed_write_test.go b/weed/s3api/s3api_object_routed_write_test.go
index ebd0602115..4986fcaac0 100644
--- a/weed/s3api/s3api_object_routed_write_test.go
+++ b/weed/s3api/s3api_object_routed_write_test.go
@@ -336,8 +336,18 @@ func startFakeFiler(t *testing.T, impl filer_pb.SeaweedFilerServer) pb.ServerAdd
filer_pb.RegisterSeaweedFilerServer(srv, impl)
go srv.Serve(lis)
t.Cleanup(srv.Stop)
- port := lis.Addr().(*net.TCPAddr).Port
- return pb.ServerAddress(fmt.Sprintf("127.0.0.1:1.%d", port))
+ return fakeFilerAddress(lis)
+}
+
+// fakeFilerAddress returns the S3-style filer address of lis. The gRPC
+// connection cache is process-wide and keyed by address, so it drops whatever
+// channel an earlier test left for this port: a failed one would fail the first
+// RPC with a stale "connection refused", and one to a stopped fake filer can
+// still look ready.
+func fakeFilerAddress(lis net.Listener) pb.ServerAddress {
+ address := pb.ServerAddress(fmt.Sprintf("127.0.0.1:1.%d", lis.Addr().(*net.TCPAddr).Port))
+ pb.InvalidateGrpcConnection(address.ToGrpcAddress())
+ return address
}
func TestRoutedDeleteSpecificVersionUsesWormCondition(t *testing.T) {
@@ -480,9 +490,42 @@ func closedFilerAddress(t *testing.T) pb.ServerAddress {
if err != nil {
t.Fatal(err)
}
- port := lis.Addr().(*net.TCPAddr).Port
lis.Close()
- return pb.ServerAddress(fmt.Sprintf("127.0.0.1:1.%d", port))
+ return fakeFilerAddress(lis)
+}
+
+// A fake filer that gets the port of a dead owner an earlier test dialed must
+// not inherit that owner's failed channel.
+func TestFakeFilerOnReusedPortIsReachable(t *testing.T) {
+ live := &fakeTxnFiler{}
+ liveAddr := startFakeFiler(t, live)
+ deadOwner := closedFilerAddress(t)
+ dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+ s3a := &S3ApiServer{
+ option: &S3ApiServerOption{GrpcDialOption: dialOption},
+ filerClient: wdclient.NewFilerClient([]pb.ServerAddress{liveAddr}, dialOption, ""),
+ }
+ if _, err := s3a.objectTxnOnFiler(deadOwner, &filer_pb.ObjectTransactionRequest{LockKey: "/buckets/b/o", RouteKey: "s3.object.write:/buckets/b/o"}); err != nil {
+ t.Fatalf("expected failover success, got %v", err)
+ }
+
+ lis, err := net.Listen("tcp", deadOwner.ToGrpcAddress())
+ if err != nil {
+ t.Skipf("port of the dead owner was taken: %v", err)
+ }
+ reused := &fakeTxnFiler{}
+ srv := grpc.NewServer()
+ filer_pb.RegisterSeaweedFilerServer(srv, reused)
+ go srv.Serve(lis)
+ t.Cleanup(srv.Stop)
+
+ direct := &S3ApiServer{option: &S3ApiServerOption{GrpcDialOption: dialOption}}
+ if _, err := direct.objectTxnOnFiler(fakeFilerAddress(lis), &filer_pb.ObjectTransactionRequest{LockKey: "/buckets/b/o"}); err != nil {
+ t.Fatalf("fake filer on a reused port: %v", err)
+ }
+ if got := atomic.LoadInt32(&reused.calls); got != 1 {
+ t.Fatalf("reused-port filer calls = %d, want 1", got)
+ }
}
// A routed write whose owner address is dead fails over to a live filer (which
diff --git a/weed/s3api/s3api_remote_storage_test.go b/weed/s3api/s3api_remote_storage_test.go
index 689ea5e59f..627a10f854 100644
--- a/weed/s3api/s3api_remote_storage_test.go
+++ b/weed/s3api/s3api_remote_storage_test.go
@@ -537,8 +537,7 @@ func startFakeCacheFiler(t *testing.T, impl *fakeCacheFiler) pb.ServerAddress {
filer_pb.RegisterSeaweedFilerServer(srv, impl)
go srv.Serve(lis)
t.Cleanup(srv.Stop)
- port := lis.Addr().(*net.TCPAddr).Port
- return pb.ServerAddress(fmt.Sprintf("127.0.0.1:1.%d", port))
+ return fakeFilerAddress(lis)
}
func newRemoteCacheTestServer(filerAddr pb.ServerAddress) *S3ApiServer {
diff --git a/weed/s3api/s3api_tables_routing_test.go b/weed/s3api/s3api_tables_routing_test.go
index 4ffbfb1e6b..e1043d8e01 100644
--- a/weed/s3api/s3api_tables_routing_test.go
+++ b/weed/s3api/s3api_tables_routing_test.go
@@ -3,7 +3,6 @@ package s3api
import (
"context"
"encoding/json"
- "fmt"
"net"
"net/http"
"net/http/httptest"
@@ -260,7 +259,7 @@ func startRecordingFiler(t *testing.T, f *recordingFiler) pb.ServerAddress {
filer_pb.RegisterSeaweedFilerServer(srv, f)
go srv.Serve(lis)
t.Cleanup(srv.Stop)
- return pb.ServerAddress(fmt.Sprintf("127.0.0.1:1.%d", lis.Addr().(*net.TCPAddr).Port))
+ return fakeFilerAddress(lis)
}
// newS3TablesServeTestServer builds the production router over a gateway whose