SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
Give the local Unix socket gRPC transport room to breathe (#10824)
Chris Luda4f06ec1204
diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.goindex f1c5bbcc9a..878775b9b7 100644--- a/weed/pb/grpc_client_server.go+++ b/weed/pb/grpc_client_server.go@@ -150,11 +150,13 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err)}- listener, err := net.Listen("unix", socketPath)+ lc := net.ListenConfig{Control: setLocalSocketBuffers}+ listener, err := lc.Listen(context.Background(), "unix", socketPath)if err != nil {glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err)return}+ listener = &localSocketListener{Listener: listener}glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)go func() {if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {@@ -164,6 +166,20 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {}()}+// localSocketListener re-applies the buffer sizes to every accepted connection.+type localSocketListener struct {+ net.Listener+}++func (l *localSocketListener) Accept() (net.Conn, error) {+ c, err := l.Listener.Accept()+ if err != nil {+ return nil, err+ }+ applyLocalSocketBuffers(c)+ return c, nil+}+func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {var options []grpc.ServerOptionoptions = append(options,@@ -209,7 +225,7 @@ func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...gr// Route through Unix socket if one is registered for this address's portif socketPath := resolveLocalGrpcSocket(address); socketPath != "" {options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {- var d net.Dialer+ d := net.Dialer{Control: setLocalSocketBuffers}return d.DialContext(ctx, "unix", socketPath)}))} else {diff --git a/weed/pb/local_socket_buffers.go b/weed/pb/local_socket_buffers.gonew file mode 100644index 0000000000..3f7f426f35--- /dev/null+++ b/weed/pb/local_socket_buffers.go@@ -0,0 +1,39 @@+//go:build !windows++package pb++import (+ "net"+ "syscall"+)++// Unix socket buffers default small and never autotune (208KB on Linux, 8KB on+// macOS). Once a buffer cannot absorb what gRPC's loopyWriter emits for the+// in-flight streams the writer blocks, both peers latch grpc-go's control-buffer+// throttle, and the connection deadlocks for good. Best effort: a value the OS+// clamps down only lowers the concurrency this survives.+const localSocketBufBytes = 8 << 20++func setLocalSocketBufferFD(fd uintptr) {+ syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, localSocketBufBytes)+ syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, localSocketBufBytes)+}++func setLocalSocketBuffers(_, _ string, c syscall.RawConn) error {+ return c.Control(setLocalSocketBufferFD)+}++// applyLocalSocketBuffers tunes an already-established connection. Linux does+// not carry the listener's buffer sizes onto accepted sockets, so the server+// half needs setting explicitly or only the dialing side gets the headroom.+func applyLocalSocketBuffers(c net.Conn) {+ sc, ok := c.(syscall.Conn)+ if !ok {+ return+ }+ raw, err := sc.SyscallConn()+ if err != nil {+ return+ }+ raw.Control(setLocalSocketBufferFD)+}diff --git a/weed/pb/local_socket_buffers_windows.go b/weed/pb/local_socket_buffers_windows.gonew file mode 100644index 0000000000..b847c5c1a3--- /dev/null+++ b/weed/pb/local_socket_buffers_windows.go@@ -0,0 +1,13 @@+//go:build windows++package pb++import (+ "net"+ "syscall"+)++// Windows never registers local Unix sockets, so there is nothing to tune.+func setLocalSocketBuffers(_, _ string, _ syscall.RawConn) error { return nil }++func applyLocalSocketBuffers(_ net.Conn) {}