SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package tasks
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
// TaskLogHandler handles task log requests from admin server
type TaskLogHandler struct {
baseLogDir string
}
// NewTaskLogHandler creates a new task log handler
func NewTaskLogHandler(baseLogDir string) *TaskLogHandler {
if baseLogDir == "" {
baseLogDir = "/tmp/seaweedfs/task_logs"
}
// Best-effort ensure the base directory exists so reads don't fail due to missing dir
if err := os.MkdirAll(baseLogDir, 0755); err != nil {
glog.Warningf("Failed to create base task log directory %s: %v", baseLogDir, err)
}
return &TaskLogHandler{
baseLogDir: baseLogDir,
}
}
// HandleLogRequest processes a task log request and returns the response
func (h *TaskLogHandler) HandleLogRequest(request *worker_pb.TaskLogRequest) *worker_pb.TaskLogResponse {
response := &worker_pb.TaskLogResponse{
TaskId: request.TaskId,
WorkerId: request.WorkerId,
Success: false,
}
// Find the task log directory
logDir, err := h.findTaskLogDirectory(request.TaskId)
if err != nil {
response.ErrorMessage = fmt.Sprintf("Task log directory not found: %v", err)
glog.Warningf("Task log request failed for %s: %v", request.TaskId, err)
// Add diagnostic information to help debug the issue
response.LogEntries = []*worker_pb.TaskLogEntry{
{
Timestamp: time.Now().Unix(),
Level: "WARNING",
Message: fmt.Sprintf("Task logs not available: %v", err),
Fields: map[string]string{"source": "task_log_handler"},
},
{
Timestamp: time.Now().Unix(),
Level: "INFO",
Message: fmt.Sprintf("This usually means the task was never executed on this worker or logs were cleaned up. Base log directory: %s", h.baseLogDir),
Fields: map[string]string{"source": "task_log_handler"},
},
}
// response.Success remains false to indicate logs were not found
return response
}
// Read metadata if requested
if request.IncludeMetadata {
metadata, err := h.readTaskMetadata(logDir)
if err != nil {
response.ErrorMessage = fmt.Sprintf("Failed to read task metadata: %v", err)
glog.Warningf("Failed to read metadata for task %s: %v", request.TaskId, err)
return response
}
response.Metadata = metadata
}
// Read log entries
logEntries, err := h.readTaskLogEntries(logDir, request)
if err != nil {
response.ErrorMessage = fmt.Sprintf("Failed to read task logs: %v", err)
glog.Warningf("Failed to read logs for task %s: %v", request.TaskId, err)
return response
}
response.LogEntries = logEntries
response.Success = true
glog.V(1).Infof("Successfully retrieved %d log entries for task %s", len(logEntries), request.TaskId)
return response
}
// findTaskLogDirectory searches for the task log directory by task ID
func (h *TaskLogHandler) findTaskLogDirectory(taskID string) (string, error) {
entries, err := os.ReadDir(h.baseLogDir)
if err != nil {
return "", fmt.Errorf("failed to read base log directory %s: %w", h.baseLogDir, err)
}
// Look for directories that start with the task ID
var candidateDirs []string
for _, entry := range entries {
if entry.IsDir() {
candidateDirs = append(candidateDirs, entry.Name())
if strings.HasPrefix(entry.Name(), taskID+"_") {
return filepath.Join(h.baseLogDir, entry.Name()), nil
}
}
}
// Enhanced error message with diagnostic information
return "", fmt.Errorf("task log directory not found for task ID: %s (searched %d directories in %s, directories found: %v)",
taskID, len(candidateDirs), h.baseLogDir, candidateDirs)
}
// readTaskMetadata reads task metadata from the log directory
func (h *TaskLogHandler) readTaskMetadata(logDir string) (*worker_pb.TaskLogMetadata, error) {
metadata, err := GetTaskLogMetadata(logDir)
if err != nil {
return nil, err
}
// Convert to protobuf metadata
pbMetadata := &worker_pb.TaskLogMetadata{
TaskId: metadata.TaskID,
TaskType: metadata.TaskType,
WorkerId: metadata.WorkerID,
StartTime: metadata.StartTime.Unix(),
Status: metadata.Status,
Progress: float32(metadata.Progress),
VolumeId: metadata.VolumeID,
Server: metadata.Server,
Collection: metadata.Collection,
LogFilePath: metadata.LogFilePath,
CreatedAt: metadata.CreatedAt.Unix(),
CustomData: make(map[string]string),
}
// Set end time and duration if available
if metadata.EndTime != nil {
pbMetadata.EndTime = metadata.EndTime.Unix()
}
if metadata.Duration != nil {
pbMetadata.DurationMs = metadata.Duration.Milliseconds()
}
// Convert custom data
for key, value := range metadata.CustomData {
if strValue, ok := value.(string); ok {
pbMetadata.CustomData[key] = strValue
} else {
pbMetadata.CustomData[key] = fmt.Sprintf("%v", value)
}
}
return pbMetadata, nil
}
// readTaskLogEntries reads and filters log entries based on the request
func (h *TaskLogHandler) readTaskLogEntries(logDir string, request *worker_pb.TaskLogRequest) ([]*worker_pb.TaskLogEntry, error) {
entries, err := ReadTaskLogs(logDir)
if err != nil {
return nil, err
}
// Apply filters
var filteredEntries []TaskLogEntry
for _, entry := range entries {
// Filter by log level
if request.LogLevel != "" && !strings.EqualFold(entry.Level, request.LogLevel) {
continue
}
// Filter by time range
if request.StartTime > 0 && entry.Timestamp.Unix() < request.StartTime {
continue
}
if request.EndTime > 0 && entry.Timestamp.Unix() > request.EndTime {
continue
}
filteredEntries = append(filteredEntries, entry)
}
// Limit entries if requested
if request.MaxEntries > 0 && len(filteredEntries) > int(request.MaxEntries) {
// Take the most recent entries
start := len(filteredEntries) - int(request.MaxEntries)
filteredEntries = filteredEntries[start:]
}
// Convert to protobuf entries
var pbEntries []*worker_pb.TaskLogEntry
for _, entry := range filteredEntries {
pbEntry := &worker_pb.TaskLogEntry{
Timestamp: entry.Timestamp.Unix(),
Level: entry.Level,
Message: entry.Message,
Fields: make(map[string]string),
}
// Set progress if available
if entry.Progress != nil {
pbEntry.Progress = float32(*entry.Progress)
}
// Set status if available
if entry.Status != nil {
pbEntry.Status = *entry.Status
}
// Convert fields
for key, value := range entry.Fields {
if strValue, ok := value.(string); ok {
pbEntry.Fields[key] = strValue
} else {
pbEntry.Fields[key] = fmt.Sprintf("%v", value)
}
}
pbEntries = append(pbEntries, pbEntry)
}
return pbEntries, nil
}