SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package plugin
import (
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// orphanedExecutionMisses is how many itemized heartbeats in a row from a
// newer stream of the worker must leave a dispatched job out before the
// admin fails it. The second rides out a heartbeat the worker built while
// it was still picking the job back up.
const orphanedExecutionMisses = 2
// pendingExecutionState is an execution request waiting for its JobCompleted.
type pendingExecutionState struct {
completed chan *plugin_pb.JobCompleted
jobID string
jobType string
// session is the stream the request went out on.
session *streamSession
misses int
failed bool
// completionSent is set once a completion is handed to the dispatcher,
// which may already have taken it from the buffer.
completionSent bool
}
// failExecutionsMissingFromHeartbeat fails the executions sent on an earlier
// stream of the heartbeat's worker that the worker no longer lists. A worker
// that restarted comes back under the same id without the old process's
// jobs, and nothing else ends such a dispatch before its execution timeout,
// so its slot stays reserved all that time. Jobs sent on the heartbeat's own
// stream are left alone, since a worker counts a job before it lists it, and
// so is everything while the heartbeat counts slots it does not itemize,
// since a missing id then proves nothing.
func (r *Plugin) failExecutionsMissingFromHeartbeat(session *streamSession, heartbeat *plugin_pb.WorkerHeartbeat) {
if session == nil || heartbeat == nil {
return
}
itemized := 0
listed := make(map[string]struct{}, len(heartbeat.GetRunningWork()))
for _, work := range heartbeat.GetRunningWork() {
if work.GetKind() == plugin_pb.WorkKind_WORK_KIND_EXECUTION {
listed[strings.TrimSpace(work.GetWorkId())] = struct{}{}
itemized++
}
}
itemizesAll := int(heartbeat.GetExecutionSlotsUsed()) <= itemized
var orphaned []activeJobSnapshot
r.pendingExecutionMu.Lock()
for requestID, pending := range r.pendingExecution {
if pending.failed || pending.session == nil || pending.session == session || pending.session.workerID != session.workerID {
continue
}
if _, ok := listed[strings.TrimSpace(pending.jobID)]; ok {
pending.misses = 0
continue
}
if !itemizesAll {
continue
}
if pending.completionSent {
continue
}
pending.misses++
if pending.misses < orphanedExecutionMisses {
continue
}
pending.failed = true
orphaned = append(orphaned, activeJobSnapshot{
jobID: pending.jobID,
jobType: pending.jobType,
workerID: session.workerID,
requestID: requestID,
})
}
r.pendingExecutionMu.Unlock()
for _, snap := range orphaned {
reason := fmt.Sprintf("worker %s re-registered without job %s", snap.workerID, snap.jobID)
glog.Warningf("Plugin failing %s request %s: %s", snap.jobType, snap.requestID, reason)
// off the stream's receive loop: the cancel may wait on a full buffer
go r.expireActiveJob(snap, reason, time.Now().UTC())
}
}