Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

Give the local Unix socket gRPC transport room to breathe (#10824)

Chris Luda4f06ec1204
diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go
index f1c5bbcc9a..878775b9b7 100644
--- a/weed/pb/grpc_client_server.go
+++ b/weed/pb/grpc_client_server.go
@@ -150,11 +150,13 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err)
}
- listener, err := net.Listen("unix", socketPath)
+ lc := net.ListenConfig{Control: setLocalSocketBuffers}
+ listener, err := lc.Listen(context.Background(), "unix", socketPath)
if err != nil {
glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err)
return
}
+ listener = &localSocketListener{Listener: listener}
glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)
go func() {
if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {
@@ -164,6 +166,20 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
}()
}
+// localSocketListener re-applies the buffer sizes to every accepted connection.
+type localSocketListener struct {
+ net.Listener
+}
+
+func (l *localSocketListener) Accept() (net.Conn, error) {
+ c, err := l.Listener.Accept()
+ if err != nil {
+ return nil, err
+ }
+ applyLocalSocketBuffers(c)
+ return c, nil
+}
+
func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
var options []grpc.ServerOption
options = append(options,
@@ -209,7 +225,7 @@ func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...gr
// Route through Unix socket if one is registered for this address's port
if socketPath := resolveLocalGrpcSocket(address); socketPath != "" {
options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
- var d net.Dialer
+ d := net.Dialer{Control: setLocalSocketBuffers}
return d.DialContext(ctx, "unix", socketPath)
}))
} else {
diff --git a/weed/pb/local_socket_buffers.go b/weed/pb/local_socket_buffers.go
new file mode 100644
index 0000000000..3f7f426f35
--- /dev/null
+++ b/weed/pb/local_socket_buffers.go
@@ -0,0 +1,39 @@
+//go:build !windows
+
+package pb
+
+import (
+ "net"
+ "syscall"
+)
+
+// Unix socket buffers default small and never autotune (208KB on Linux, 8KB on
+// macOS). Once a buffer cannot absorb what gRPC's loopyWriter emits for the
+// in-flight streams the writer blocks, both peers latch grpc-go's control-buffer
+// throttle, and the connection deadlocks for good. Best effort: a value the OS
+// clamps down only lowers the concurrency this survives.
+const localSocketBufBytes = 8 << 20
+
+func setLocalSocketBufferFD(fd uintptr) {
+ syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, localSocketBufBytes)
+ syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, localSocketBufBytes)
+}
+
+func setLocalSocketBuffers(_, _ string, c syscall.RawConn) error {
+ return c.Control(setLocalSocketBufferFD)
+}
+
+// applyLocalSocketBuffers tunes an already-established connection. Linux does
+// not carry the listener's buffer sizes onto accepted sockets, so the server
+// half needs setting explicitly or only the dialing side gets the headroom.
+func applyLocalSocketBuffers(c net.Conn) {
+ sc, ok := c.(syscall.Conn)
+ if !ok {
+ return
+ }
+ raw, err := sc.SyscallConn()
+ if err != nil {
+ return
+ }
+ raw.Control(setLocalSocketBufferFD)
+}
diff --git a/weed/pb/local_socket_buffers_windows.go b/weed/pb/local_socket_buffers_windows.go
new file mode 100644
index 0000000000..b847c5c1a3
--- /dev/null
+++ b/weed/pb/local_socket_buffers_windows.go
@@ -0,0 +1,13 @@
+//go:build windows
+
+package pb
+
+import (
+ "net"
+ "syscall"
+)
+
+// Windows never registers local Unix sockets, so there is nothing to tune.
+func setLocalSocketBuffers(_, _ string, _ syscall.RawConn) error { return nil }
+
+func applyLocalSocketBuffers(_ net.Conn) {}