Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

26.3 KB
Raw Blame
package dash

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
	"github.com/seaweedfs/seaweedfs/weed/security"
	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/status"
)

const (
	maxLogFetchLimit   = 1000
	maxLogMessageSize  = 2000
	maxLogFieldsCount  = 20
	logRequestTimeout  = 10 * time.Second
	logResponseTimeout = 30 * time.Second
	logSendTimeout     = 10 * time.Second
)

// WorkerGrpcServer implements the WorkerService gRPC interface
type WorkerGrpcServer struct {
	worker_pb.UnimplementedWorkerServiceServer
	adminServer *AdminServer

	// Worker connection management
	connections map[string]*WorkerConnection
	connMutex   sync.RWMutex

	// Log request correlation
	pendingLogRequests map[string]*LogRequestContext
	logRequestsMutex   sync.RWMutex

	// gRPC server
	grpcServer  *grpc.Server
	listener    net.Listener
	running     bool
	stopChan    chan struct{}
	mtlsEnabled bool
}

// LogRequestContext tracks pending log requests
type LogRequestContext struct {
	TaskID     string
	WorkerID   string
	ResponseCh chan *worker_pb.TaskLogResponse
}

// WorkerConnection represents an active worker connection
type WorkerConnection struct {
	workerID      string
	stream        worker_pb.WorkerService_WorkerStreamServer
	lastSeen      time.Time
	capabilities  []MaintenanceTaskType
	address       string
	maxConcurrent int32
	outgoing      chan *worker_pb.AdminMessage
	ctx           context.Context
	cancel        context.CancelFunc
}

// NewWorkerGrpcServer creates a new gRPC server for worker connections
func NewWorkerGrpcServer(adminServer *AdminServer) *WorkerGrpcServer {
	return &WorkerGrpcServer{
		adminServer:        adminServer,
		connections:        make(map[string]*WorkerConnection),
		pendingLogRequests: make(map[string]*LogRequestContext),
		stopChan:           make(chan struct{}),
	}
}

// StartWithTLS starts the gRPC server on the specified port with optional TLS.
// A caller that already holds the port passes its listener instead. When no
// listener is supplied the server binds to bindIp to honor the operator's -ip.
func (s *WorkerGrpcServer) StartWithTLS(bindIp string, port int, listener net.Listener) error {
	if s.running {
		return fmt.Errorf("worker gRPC server is already running")
	}

	if listener == nil {
		var err error
		listener, err = net.Listen("tcp", util.JoinHostPort(bindIp, port))
		if err != nil {
			return fmt.Errorf("failed to listen on port %d: %v", port, err)
		}
	}

	// Create gRPC server with optional TLS
	tlsOption, _ := security.LoadServerTLS(util.GetViper(), "grpc.admin")
	s.mtlsEnabled = tlsOption != nil
	grpcServer := pb.NewGrpcServer(tlsOption)

	worker_pb.RegisterWorkerServiceServer(grpcServer, s)
	if plugin := s.adminServer.GetPlugin(); plugin != nil {
		plugin_pb.RegisterPluginControlServiceServer(grpcServer, plugin)
		glog.V(0).Infof("Plugin gRPC service registered on worker gRPC server")
	}

	s.grpcServer = grpcServer
	s.listener = listener
	s.running = true

	// Start background routines
	go s.cleanupRoutine()
	go s.activeLogFetchLoop()

	pb.ServeGrpcOnLocalSocket(grpcServer, port)

	// Start serving in a goroutine
	go func() {
		if err := s.grpcServer.Serve(listener); err != nil {
			if s.running {
				glog.Errorf("Worker gRPC server error: %v", err)
			}
		}
	}()

	return nil
}

// ServeOn serves the worker gRPC API on an additional already-bound listener,
// e.g. a legacy compat port kept alongside the primary one. grpc.Server tracks
// every listener passed to Serve, so Stop closes it together with the primary.
func (s *WorkerGrpcServer) ServeOn(listener net.Listener) {
	go func() {
		if err := s.grpcServer.Serve(listener); err != nil && s.running {
			glog.Warningf("Worker gRPC compat listener on %s stopped: %v", listener.Addr(), err)
		}
	}()
}

// ListenPort returns the currently bound worker gRPC listen port.
func (s *WorkerGrpcServer) ListenPort() int {
	if s == nil || s.listener == nil {
		return 0
	}
	if tcpAddr, ok := s.listener.Addr().(*net.TCPAddr); ok {
		return tcpAddr.Port
	}
	_, portStr, err := net.SplitHostPort(s.listener.Addr().String())
	if err != nil {
		return 0
	}
	port, err := strconv.Atoi(portStr)
	if err != nil {
		return 0
	}
	return port
}

// Stop stops the gRPC server
func (s *WorkerGrpcServer) Stop() error {
	if !s.running {
		return nil
	}

	s.running = false
	close(s.stopChan)

	// Close all worker connections. Cancelling the context stops each
	// handleOutgoingMessages goroutine; the outgoing channel is never closed.
	s.connMutex.Lock()
	for _, conn := range s.connections {
		conn.cancel()
	}
	s.connections = make(map[string]*WorkerConnection)
	s.connMutex.Unlock()

	// Stop gRPC server
	if s.grpcServer != nil {
		s.grpcServer.GracefulStop()
	}

	// Close listener
	if s.listener != nil {
		s.listener.Close()
	}

	glog.Infof("Worker gRPC server stopped")
	return nil
}

// WorkerStream handles bidirectional communication with workers
func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStreamServer) error {
	ctx := stream.Context()

	if err := security.CheckGrpcAdminPeer(ctx, security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key"))); err != nil {
		return err
	}

	// get client address
	address := findClientAddress(ctx)

	// Wait for initial registration message
	msg, err := stream.Recv()
	if err != nil {
		return fmt.Errorf("failed to receive registration message: %w", err)
	}

	registration := msg.GetRegistration()
	if registration == nil {
		return fmt.Errorf("first message must be registration")
	}
	registration.Address = address

	workerID := registration.WorkerId
	if workerID == "" {
		return fmt.Errorf("worker ID cannot be empty")
	}

	glog.Infof("Worker %s connecting from %s", workerID, registration.Address)

	// Create worker connection
	connCtx, connCancel := context.WithCancel(ctx)
	conn := &WorkerConnection{
		workerID:      workerID,
		stream:        stream,
		lastSeen:      time.Now(),
		address:       registration.Address,
		maxConcurrent: registration.MaxConcurrent,
		outgoing:      make(chan *worker_pb.AdminMessage, 100),
		ctx:           connCtx,
		cancel:        connCancel,
	}

	// Convert capabilities
	capabilities := make([]MaintenanceTaskType, len(registration.Capabilities))
	for i, cap := range registration.Capabilities {
		capabilities[i] = MaintenanceTaskType(cap)
	}
	conn.capabilities = capabilities

	// Register connection - clean up old connection if worker is reconnecting
	s.connMutex.Lock()
	if oldConn, exists := s.connections[workerID]; exists {
		glog.Infof("Worker %s reconnected, cleaning up old connection", workerID)
		// Cancel old connection to stop its goroutines. Its handleOutgoingMessages
		// exits on the cancelled context; the outgoing channel is never closed.
		oldConn.cancel()
	}
	s.connections[workerID] = conn
	s.connMutex.Unlock()
	stats_collect.AdminWorkerEventsTotal.WithLabelValues("registered").Inc()

	// Register worker with maintenance manager
	s.registerWorkerWithManager(conn)

	// IMPORTANT: Start outgoing message handler BEFORE sending registration response
	// This ensures the handler is ready to process messages and prevents race conditions
	// where the worker might send requests before we're ready to respond
	go s.handleOutgoingMessages(conn)

	// Send registration response (after handler is started)
	regResponse := &worker_pb.AdminMessage{
		Timestamp: time.Now().Unix(),
		Message: &worker_pb.AdminMessage_RegistrationResponse{
			RegistrationResponse: &worker_pb.RegistrationResponse{
				Success: true,
				Message: "Worker registered successfully",
			},
		},
	}

	if s.sendToWorker(conn, regResponse, 5*time.Second, "registration response") {
		glog.V(1).Infof("Registration response sent to worker %s", workerID)
	}

	// Handle incoming messages
	for {
		select {
		case <-ctx.Done():
			glog.Infof("Worker %s connection closed: %v", workerID, ctx.Err())
			s.unregisterWorker(conn, "unregistered")
			return nil
		case <-connCtx.Done():
			glog.Infof("Worker %s connection cancelled", workerID)
			s.unregisterWorker(conn, "unregistered")
			return nil
		default:
		}

		msg, err := stream.Recv()
		if err != nil {
			switch {
			case err == io.EOF:
				glog.Infof("Worker %s disconnected", workerID)
			case errors.Is(err, context.Canceled), status.Code(err) == codes.Canceled:
				// Graceful shutdown on either side cancels the stream.
				glog.V(1).Infof("Worker %s stream canceled: %v", workerID, err)
			default:
				glog.Errorf("Error receiving from worker %s: %v", workerID, err)
			}
			s.unregisterWorker(conn, "unregistered")
			return err
		}

		s.connMutex.Lock()
		conn.lastSeen = time.Now()
		s.connMutex.Unlock()
		s.handleWorkerMessage(conn, msg)
	}
}

// handleOutgoingMessages sends messages to worker
func (s *WorkerGrpcServer) handleOutgoingMessages(conn *WorkerConnection) {
	for {
		select {
		case <-conn.ctx.Done():
			return
		case msg, ok := <-conn.outgoing:
			if !ok {
				return
			}

			if err := conn.stream.Send(msg); err != nil {
				glog.Errorf("Failed to send message to worker %s: %v", conn.workerID, err)
				conn.cancel()
				return
			}
		}
	}
}

// handleWorkerMessage processes incoming messages from workers
func (s *WorkerGrpcServer) handleWorkerMessage(conn *WorkerConnection, msg *worker_pb.WorkerMessage) {
	workerID := conn.workerID

	switch m := msg.Message.(type) {
	case *worker_pb.WorkerMessage_Heartbeat:
		s.handleHeartbeat(conn, m.Heartbeat)

	case *worker_pb.WorkerMessage_TaskRequest:
		s.handleTaskRequest(conn, m.TaskRequest)

	case *worker_pb.WorkerMessage_TaskUpdate:
		s.handleTaskUpdate(conn, m.TaskUpdate)

	case *worker_pb.WorkerMessage_TaskComplete:
		s.handleTaskCompletion(conn, m.TaskComplete)

	case *worker_pb.WorkerMessage_TaskLogResponse:
		s.handleTaskLogResponse(conn, m.TaskLogResponse)

	case *worker_pb.WorkerMessage_Shutdown:
		glog.Infof("Worker %s shutting down: %s", workerID, m.Shutdown.Reason)
		s.unregisterWorker(conn, "unregistered")

	default:
		glog.Warningf("Unknown message type from worker %s", workerID)
	}
}

// registerWorkerWithManager registers the worker with the maintenance manager
func (s *WorkerGrpcServer) registerWorkerWithManager(conn *WorkerConnection) {
	if s.adminServer.maintenanceManager == nil {
		return
	}

	worker := &MaintenanceWorker{
		ID:            conn.workerID,
		Address:       conn.address,
		LastHeartbeat: time.Now(),
		Status:        "active",
		Capabilities:  conn.capabilities,
		MaxConcurrent: int(conn.maxConcurrent),
		CurrentLoad:   0,
	}

	s.adminServer.maintenanceManager.RegisterWorker(worker)
	glog.V(1).Infof("Registered worker %s with maintenance manager", conn.workerID)
}

// handleHeartbeat processes heartbeat messages
func (s *WorkerGrpcServer) handleHeartbeat(conn *WorkerConnection, heartbeat *worker_pb.WorkerHeartbeat) {
	if s.adminServer.maintenanceManager != nil {
		s.adminServer.maintenanceManager.UpdateWorkerHeartbeat(conn.workerID)
	}

	// Send heartbeat response
	response := &worker_pb.AdminMessage{
		Timestamp: time.Now().Unix(),
		Message: &worker_pb.AdminMessage_HeartbeatResponse{
			HeartbeatResponse: &worker_pb.HeartbeatResponse{
				Success: true,
				Message: "Heartbeat acknowledged",
			},
		},
	}

	s.sendToWorker(conn, response, time.Second, "heartbeat response")
}

// handleTaskRequest processes task requests from workers
func (s *WorkerGrpcServer) handleTaskRequest(conn *WorkerConnection, request *worker_pb.TaskRequest) {

	if s.adminServer.maintenanceManager == nil {
		return
	}

	// Get next task from maintenance manager
	task := s.adminServer.maintenanceManager.GetNextTask(conn.workerID, conn.capabilities)

	if task != nil {

		// Use typed params directly - master client should already be configured in the params
		var taskParams *worker_pb.TaskParams
		if task.TypedParams != nil {
			taskParams = task.TypedParams
		} else {
			// Create basic params if none exist
			taskParams = &worker_pb.TaskParams{
				VolumeId:   task.VolumeID,
				Collection: task.Collection,
				Sources: []*worker_pb.TaskSource{
					{
						Node:     task.Server,
						VolumeId: task.VolumeID,
					},
				},
			}
		}

		// Send task assignment
		assignment := &worker_pb.AdminMessage{
			Timestamp: time.Now().Unix(),
			Message: &worker_pb.AdminMessage_TaskAssignment{
				TaskAssignment: &worker_pb.TaskAssignment{
					TaskId:      task.ID,
					TaskType:    string(task.Type),
					Params:      taskParams,
					Priority:    int32(task.Priority),
					CreatedTime: time.Now().Unix(),
				},
			},
		}

		s.sendToWorker(conn, assignment, time.Second, "task assignment")
	} else {
		// Send explicit "No Task" response to prevent worker timeout
		// Workers expect a TaskAssignment message but will sleep if TaskId is empty
		noTaskAssignment := &worker_pb.AdminMessage{
			Timestamp: time.Now().Unix(),
			Message: &worker_pb.AdminMessage_TaskAssignment{
				TaskAssignment: &worker_pb.TaskAssignment{
					TaskId: "", // Empty TaskId indicates no task available
				},
			},
		}

		if s.sendToWorker(conn, noTaskAssignment, time.Second, "no-task response") {
			glog.V(4).Infof("Sent 'No Task' response to worker %s", conn.workerID)
		}
	}
}

// handleTaskUpdate processes task progress updates
func (s *WorkerGrpcServer) handleTaskUpdate(conn *WorkerConnection, update *worker_pb.TaskUpdate) {
	if s.adminServer.maintenanceManager != nil {
		s.adminServer.maintenanceManager.UpdateTaskProgress(update.TaskId, float64(update.Progress))
		glog.V(3).Infof("Updated task %s progress: %.1f%%", update.TaskId, update.Progress)
	}
}

// handleTaskCompletion processes task completion notifications
func (s *WorkerGrpcServer) handleTaskCompletion(conn *WorkerConnection, completion *worker_pb.TaskComplete) {
	if s.adminServer.maintenanceManager != nil {
		errorMsg := ""
		if !completion.Success {
			errorMsg = completion.ErrorMessage
		}
		s.adminServer.maintenanceManager.CompleteTask(completion.TaskId, errorMsg)

		if completion.Success {
			glog.V(1).Infof("Worker %s completed task %s successfully", conn.workerID, completion.TaskId)
		} else {
			glog.Errorf("Worker %s failed task %s: %s", conn.workerID, completion.TaskId, completion.ErrorMessage)
		}

		// Fetch and persist logs
		go s.FetchAndSaveLogs(conn.workerID, completion.TaskId)
	}
}

// FetchAndSaveLogs retrieves logs from a worker and saves them to disk
func (s *WorkerGrpcServer) FetchAndSaveLogs(workerID, taskID string) error {
	// Add a small initial delay to allow worker to finalize and sync logs
	// especially when this is called immediately after TaskComplete
	time.Sleep(300 * time.Millisecond)

	var workerLogs []*worker_pb.TaskLogEntry
	var err error

	// Retry a few times if fetch fails, as logs might be in the middle of a terminal sync
	for attempt := 1; attempt <= 3; attempt++ {
		workerLogs, err = s.RequestTaskLogs(workerID, taskID, maxLogFetchLimit, "")
		if err == nil {
			break
		}
		if attempt < 3 {
			glog.V(1).Infof("Fetch logs attempt %d failed for task %s: %v. Retrying in 1s...", attempt, taskID, err)
			time.Sleep(1 * time.Second)
		}
	}

	if err != nil {
		glog.Warningf("Failed to fetch logs for task %s after 3 attempts: %v", taskID, err)
		return err
	}

	// Convert logs
	var maintenanceLogs []*maintenance.TaskExecutionLog
	for _, workerLog := range workerLogs {
		maintenanceLog := &maintenance.TaskExecutionLog{
			Timestamp: time.Unix(workerLog.Timestamp, 0),
			Level:     workerLog.Level,
			Message:   workerLog.Message,
			Source:    "worker",
			TaskID:    taskID,
			WorkerID:  workerID,
		}

		// Truncate very long messages to prevent rendering issues and disk bloat
		if len(maintenanceLog.Message) > maxLogMessageSize {
			maintenanceLog.Message = maintenanceLog.Message[:maxLogMessageSize] + "... (truncated)"
		}

		// carry structured fields if present
		if len(workerLog.Fields) > 0 {
			maintenanceLog.Fields = make(map[string]string)
			fieldCount := 0
			for k, v := range workerLog.Fields {
				if fieldCount >= maxLogFieldsCount {
					maintenanceLog.Fields["..."] = fmt.Sprintf("(%d more fields truncated)", len(workerLog.Fields)-maxLogFieldsCount)
					break
				}
				maintenanceLog.Fields[k] = v
				fieldCount++
			}
		}

		// carry optional progress/status
		if workerLog.Progress != 0 {
			p := float64(workerLog.Progress)
			maintenanceLog.Progress = &p
		}
		if workerLog.Status != "" {
			maintenanceLog.Status = workerLog.Status
		}
		maintenanceLogs = append(maintenanceLogs, maintenanceLog)
	}

	// Persist logs
	if s.adminServer.configPersistence != nil {
		if err := s.adminServer.configPersistence.SaveTaskExecutionLogs(taskID, maintenanceLogs); err != nil {
			glog.Errorf("Failed to persist logs for task %s: %v", taskID, err)
			return err
		}
	}
	return nil
}

// handleTaskLogResponse processes task log responses from workers
func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, response *worker_pb.TaskLogResponse) {
	requestKey := fmt.Sprintf("%s:%s", response.WorkerId, response.TaskId)

	s.logRequestsMutex.RLock()
	requestContext, exists := s.pendingLogRequests[requestKey]
	s.logRequestsMutex.RUnlock()

	if !exists {
		glog.Warningf("Received unexpected log response for task %s from worker %s", response.TaskId, response.WorkerId)
		return
	}

	glog.V(1).Infof("Received log response for task %s from worker %s: %d entries", response.TaskId, response.WorkerId, len(response.LogEntries))

	// Send response to waiting channel
	select {
	case requestContext.ResponseCh <- response:
		// Response delivered successfully
	case <-time.After(time.Second):
		glog.Warningf("Failed to deliver log response for task %s from worker %s: timeout", response.TaskId, response.WorkerId)
	}

	// Clean up the pending request
	s.logRequestsMutex.Lock()
	delete(s.pendingLogRequests, requestKey)
	s.logRequestsMutex.Unlock()
}

// sendToWorker queues a message on conn.outgoing, which is deliberately never
// closed: it has multiple concurrent senders, so closing it could panic one of
// them with "send on closed channel". Teardown is signaled via conn.ctx instead;
// handleOutgoingMessages drains the channel until that context is cancelled.
// Returns false if the connection closed or the send timed out.
func (s *WorkerGrpcServer) sendToWorker(conn *WorkerConnection, msg *worker_pb.AdminMessage, timeout time.Duration, description string) bool {
	select {
	case conn.outgoing <- msg:
		return true
	case <-conn.ctx.Done():
		glog.V(2).Infof("Dropped %s for worker %s: connection closed", description, conn.workerID)
		return false
	case <-time.After(timeout):
		glog.Warningf("Failed to send %s to worker %s: timeout", description, conn.workerID)
		return false
	}
}

// unregisterWorker removes a worker connection
func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection, event string) {
	s.connMutex.Lock()
	existingConn, exists := s.connections[conn.workerID]
	if !exists {
		s.connMutex.Unlock()
		glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", conn.workerID)
		return
	}

	// Only remove if it matches the specific connection instance
	if existingConn != conn {
		s.connMutex.Unlock()
		glog.V(1).Infof("unregisterWorker: worker %s connection replaced, skipping unregister for old connection", conn.workerID)
		return
	}

	// Remove from map first to prevent duplicate cleanup attempts
	delete(s.connections, conn.workerID)
	s.connMutex.Unlock()
	stats_collect.AdminWorkerEventsTotal.WithLabelValues(event).Inc()

	// Cancel context to signal goroutines to stop. The outgoing channel is
	// never closed (it has multiple senders); handleOutgoingMessages exits on
	// the cancelled context.
	conn.cancel()

	glog.V(1).Infof("Unregistered worker %s", conn.workerID)
}

// cleanupRoutine periodically cleans up stale connections
func (s *WorkerGrpcServer) cleanupRoutine() {
	ticker := time.NewTicker(30 * time.Second)
	defer ticker.Stop()

	for {
		select {
		case <-s.stopChan:
			return
		case <-ticker.C:
			s.cleanupStaleConnections()
		}
	}
}

// cleanupStaleConnections removes connections that haven't been seen recently
func (s *WorkerGrpcServer) cleanupStaleConnections() {
	cutoff := time.Now().Add(-2 * time.Minute)

	s.connMutex.Lock()
	// collect connections to remove first to avoid deadlock if unregisterWorker locks
	var toRemove []*WorkerConnection
	for _, conn := range s.connections {
		if conn.lastSeen.Before(cutoff) {
			toRemove = append(toRemove, conn)
		}
	}
	s.connMutex.Unlock()

	for _, conn := range toRemove {
		glog.Warningf("Cleaning up stale worker connection: %s", conn.workerID)
		s.unregisterWorker(conn, "stale_removed")
	}
}

// GetConnectedWorkers returns a list of currently connected workers
func (s *WorkerGrpcServer) GetConnectedWorkers() []string {
	s.connMutex.RLock()
	defer s.connMutex.RUnlock()

	workers := make([]string, 0, len(s.connections))
	for workerID := range s.connections {
		workers = append(workers, workerID)
	}
	return workers
}

// RequestTaskLogs requests execution logs from a worker for a specific task
func (s *WorkerGrpcServer) RequestTaskLogs(workerID, taskID string, maxEntries int32, logLevel string) ([]*worker_pb.TaskLogEntry, error) {
	s.connMutex.RLock()
	conn, exists := s.connections[workerID]
	s.connMutex.RUnlock()

	if !exists {
		return nil, fmt.Errorf("worker %s is not connected", workerID)
	}

	// Create response channel for this request
	responseCh := make(chan *worker_pb.TaskLogResponse, 1)
	requestKey := fmt.Sprintf("%s:%s", workerID, taskID)

	// Register pending request
	requestContext := &LogRequestContext{
		TaskID:     taskID,
		WorkerID:   workerID,
		ResponseCh: responseCh,
	}

	s.logRequestsMutex.Lock()
	if _, exists := s.pendingLogRequests[requestKey]; exists {
		s.logRequestsMutex.Unlock()
		return nil, fmt.Errorf("a log request for task %s is already in progress", taskID)
	}
	s.pendingLogRequests[requestKey] = requestContext
	s.logRequestsMutex.Unlock()

	// Create log request message
	logRequest := &worker_pb.AdminMessage{
		AdminId:   "admin-server",
		Timestamp: time.Now().Unix(),
		Message: &worker_pb.AdminMessage_TaskLogRequest{
			TaskLogRequest: &worker_pb.TaskLogRequest{
				TaskId:          taskID,
				WorkerId:        workerID,
				IncludeMetadata: true,
				MaxEntries:      maxEntries,
				LogLevel:        logLevel,
			},
		},
	}

	// Send the request through the worker's outgoing channel
	select {
	case conn.outgoing <- logRequest:
		glog.V(1).Infof("Log request sent to worker %s for task %s", workerID, taskID)
	case <-conn.ctx.Done():
		s.logRequestsMutex.Lock()
		if s.pendingLogRequests[requestKey] == requestContext {
			delete(s.pendingLogRequests, requestKey)
		}
		s.logRequestsMutex.Unlock()
		return nil, fmt.Errorf("worker %s connection closed", workerID)
	case <-time.After(logSendTimeout):
		// Clean up pending request on timeout
		s.logRequestsMutex.Lock()
		if s.pendingLogRequests[requestKey] == requestContext {
			delete(s.pendingLogRequests, requestKey)
		}
		s.logRequestsMutex.Unlock()
		return nil, fmt.Errorf("timeout sending log request to worker %s", workerID)
	}

	// Wait for response
	select {
	case response := <-responseCh:
		if !response.Success {
			return nil, fmt.Errorf("worker log request failed: %s", response.ErrorMessage)
		}
		glog.V(1).Infof("Received %d log entries for task %s from worker %s", len(response.LogEntries), taskID, workerID)
		return response.LogEntries, nil
	case <-conn.ctx.Done():
		s.logRequestsMutex.Lock()
		if s.pendingLogRequests[requestKey] == requestContext {
			delete(s.pendingLogRequests, requestKey)
		}
		s.logRequestsMutex.Unlock()
		return nil, fmt.Errorf("worker %s connection closed", workerID)
	case <-time.After(logResponseTimeout):
		// Clean up pending request on timeout
		s.logRequestsMutex.Lock()
		if s.pendingLogRequests[requestKey] == requestContext {
			delete(s.pendingLogRequests, requestKey)
		}
		s.logRequestsMutex.Unlock()
		return nil, fmt.Errorf("timeout waiting for log response from worker %s", workerID)
	}
}

// RequestTaskLogsFromAllWorkers requests logs for a task from all connected workers
func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntries int32, logLevel string) (map[string][]*worker_pb.TaskLogEntry, error) {
	s.connMutex.RLock()
	workerIDs := make([]string, 0, len(s.connections))
	for workerID := range s.connections {
		workerIDs = append(workerIDs, workerID)
	}
	s.connMutex.RUnlock()

	results := make(map[string][]*worker_pb.TaskLogEntry)

	for _, workerID := range workerIDs {
		logs, err := s.RequestTaskLogs(workerID, taskID, maxEntries, logLevel)
		if err != nil {
			glog.V(1).Infof("Failed to get logs from worker %s for task %s: %v", workerID, taskID, err)
			// Store empty result with error information for debugging
			results[workerID+"_error"] = []*worker_pb.TaskLogEntry{
				{
					Timestamp: time.Now().Unix(),
					Level:     "ERROR",
					Message:   fmt.Sprintf("Failed to retrieve logs from worker %s: %v", workerID, err),
					Fields:    map[string]string{"source": "admin"},
				},
			}
			continue
		}
		if len(logs) > 0 {
			results[workerID] = logs
		} else {
			glog.V(2).Infof("No logs found for task %s on worker %s", taskID, workerID)
		}
	}

	return results, nil
}

func findClientAddress(ctx context.Context) string {
	// fmt.Printf("FromContext %+v\n", ctx)
	pr, ok := peer.FromContext(ctx)
	if !ok {
		glog.Error("failed to get peer from ctx")
		return ""
	}
	if pr.Addr == net.Addr(nil) {
		glog.Error("failed to get peer address")
		return ""
	}
	return pr.Addr.String()
}

// activeLogFetchLoop periodically fetches logs for all in-progress tasks
func (s *WorkerGrpcServer) activeLogFetchLoop() {
	ticker := time.NewTicker(30 * time.Second)
	defer ticker.Stop()

	for {
		select {
		case <-s.stopChan:
			return
		case <-ticker.C:
			if !s.running || s.adminServer == nil || s.adminServer.maintenanceManager == nil {
				continue
			}

			// Get all in-progress tasks
			tasks := s.adminServer.maintenanceManager.GetTasks(maintenance.TaskStatusInProgress, "", 0)
			if len(tasks) == 0 {
				continue
			}

			glog.V(2).Infof("Background log fetcher: found %d in-progress tasks", len(tasks))
			for _, task := range tasks {
				if task.WorkerID != "" {
					// Use a goroutine to avoid blocking the loop
					go func(wID, tID string) {
						if err := s.FetchAndSaveLogs(wID, tID); err != nil {
							glog.V(2).Infof("Background log fetch failed for task %s on worker %s: %v", tID, wID, err)
						}
					}(task.WorkerID, task.ID)
				}
			}
		}
	}
}