Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

36.3 KB
Raw Blame
package plugin

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/types/known/timestamppb"
)

const (
	maxTrackedJobsTotal = 1000
	maxActivityRecords  = 4000
	maxRelatedJobs      = 100

	// stale active jobs block dedupe and scheduling; use generous defaults to
	// avoid expiring legitimate long-running tasks.
	defaultStaleActiveJobTimeout    = 24 * time.Hour
	defaultOrphanedActiveJobTimeout = 15 * time.Minute
)

var (
	StateSucceeded = strings.ToLower(plugin_pb.JobState_JOB_STATE_SUCCEEDED.String())
	StateFailed    = strings.ToLower(plugin_pb.JobState_JOB_STATE_FAILED.String())
	StateCanceled  = strings.ToLower(plugin_pb.JobState_JOB_STATE_CANCELED.String())
)

type activeJobSnapshot struct {
	jobID      string
	jobType    string
	workerID   string
	requestID  string
	lastUpdate time.Time
}

// activityLess reports whether activity a occurred after activity b (newest-first order).
// A nil OccurredAt is treated as the zero time.
func activityLess(a, b JobActivity) bool {
	ta := time.Time{}
	if a.OccurredAt != nil {
		ta = *a.OccurredAt
	}
	tb := time.Time{}
	if b.OccurredAt != nil {
		tb = *b.OccurredAt
	}
	return ta.After(tb)
}

func (r *Plugin) loadPersistedMonitorState() error {
	trackedJobs, err := r.store.LoadTrackedJobs()
	if err != nil {
		return err
	}
	activities, err := r.store.LoadActivities()
	if err != nil {
		return err
	}

	if len(trackedJobs) > 0 {
		// Collect jobs that still carry inline rich details (old format) so we
		// can migrate them to dedicated per-job detail files asynchronously.
		// Writing many files synchronously during startup can be extremely slow
		// on distributed storage (e.g. Longhorn), so we defer those writes.
		var jobsToMigrate []TrackedJob

		r.jobsMu.Lock()
		for i := range trackedJobs {
			job := trackedJobs[i]
			if strings.TrimSpace(job.JobID) == "" {
				continue
			}
			if isActiveTrackedJobState(job.State) {
				if detail, detailErr := r.store.LoadJobDetail(job.JobID); detailErr != nil {
					glog.Warningf("Plugin failed to load detail snapshot for job %s: %v", job.JobID, detailErr)
				} else if detail != nil {
					mergeTerminalDetailIntoTracked(&job, detail)
				}
			}
			// Backward compatibility: migrate older inline detail payloads
			// out of tracked_jobs.json into dedicated per-job detail files.
			// Collect for async migration instead of writing synchronously.
			if hasTrackedJobRichDetails(job) {
				jobsToMigrate = append(jobsToMigrate, job)
			}
			stripTrackedJobDetailFields(&job)
			jobCopy := job
			r.jobs[job.JobID] = &jobCopy
		}
		r.pruneTrackedJobsLocked()
		r.jobsMu.Unlock()

		if len(jobsToMigrate) > 0 {
			go r.migrateInlineJobDetails(jobsToMigrate)
		}
	}

	if len(activities) > maxActivityRecords {
		activities = activities[len(activities)-maxActivityRecords:]
	}
	if len(activities) > 0 {
		r.activitiesMu.Lock()
		r.activities = append([]JobActivity(nil), activities...)
		r.activitiesMu.Unlock()
	}

	return nil
}

// migrateInlineJobDetails writes inline detail payloads from old-format
// tracked_jobs.json entries to dedicated per-job detail files. It is called
// asynchronously during startup to avoid blocking plugin initialisation.
func (r *Plugin) migrateInlineJobDetails(jobs []TrackedJob) {
	migrated := 0
	for _, job := range jobs {
		if err := r.store.SaveJobDetail(job); err != nil {
			glog.Warningf("Plugin failed to migrate detail snapshot for job %s: %v", job.JobID, err)
		} else {
			migrated++
		}
	}
	if migrated > 0 {
		glog.V(1).Infof("Plugin migrated %d inline job details to dedicated files", migrated)
	}
}

// ExpireJob marks an active job as failed so it no longer blocks scheduling.
func (r *Plugin) ExpireJob(jobID, reason string) (*TrackedJob, bool, error) {
	normalizedJobID := strings.TrimSpace(jobID)
	if normalizedJobID == "" {
		return nil, false, ErrJobNotFound
	}

	reason = strings.TrimSpace(reason)
	if reason == "" {
		reason = "job expired by admin request"
	}

	var jobType string
	var requestID string
	var workerID string
	active := false

	r.jobsMu.RLock()
	if tracked := r.jobs[normalizedJobID]; tracked != nil {
		jobType = tracked.JobType
		requestID = tracked.RequestID
		workerID = tracked.WorkerID
		active = isActiveTrackedJobState(tracked.State)
	}
	r.jobsMu.RUnlock()

	if jobType == "" || requestID == "" || !active {
		if detail, err := r.store.LoadJobDetail(normalizedJobID); err != nil {
			return nil, false, err
		} else if detail != nil {
			if jobType == "" {
				jobType = detail.JobType
			}
			if requestID == "" {
				requestID = detail.RequestID
			}
			if workerID == "" {
				workerID = detail.WorkerID
			}
			if !active && isActiveTrackedJobState(detail.State) {
				active = true
			}
		}
	}

	if jobType == "" {
		return nil, false, ErrJobNotFound
	}

	if !active {
		current, _ := r.GetTrackedJob(normalizedJobID)
		if current == nil {
			if detail, err := r.store.LoadJobDetail(normalizedJobID); err == nil && detail != nil {
				clone := cloneTrackedJob(*detail)
				current = &clone
			}
		}
		return current, false, nil
	}

	now := time.Now().UTC()
	r.expireActiveJob(activeJobSnapshot{
		jobID:     normalizedJobID,
		jobType:   jobType,
		workerID:  workerID,
		requestID: requestID,
	}, reason, now)
	r.appendActivity(JobActivity{
		JobID:      normalizedJobID,
		JobType:    jobType,
		RequestID:  requestID,
		Source:     "admin_expire",
		Message:    reason,
		Stage:      "expired",
		OccurredAt: timeToPtr(now),
	})

	updated, _ := r.GetTrackedJob(normalizedJobID)
	return updated, true, nil
}

// expireStaleJobs marks stale active jobs as failed so they stop blocking new work.
func (r *Plugin) expireStaleJobs(now time.Time) int {
	if now.IsZero() {
		now = time.Now().UTC()
	}

	r.staleJobsMu.Lock()
	defer r.staleJobsMu.Unlock()

	snapshots := r.snapshotActiveJobs()
	if len(snapshots) == 0 {
		return 0
	}

	expired := 0
	for _, snap := range snapshots {
		if snap.lastUpdate.IsZero() {
			continue
		}
		if stale, _, _ := r.evaluateStaleJob(now, snap.workerID, snap.lastUpdate); !stale {
			continue
		}

		reason := r.confirmStaleReason(now, snap.jobID)
		if reason == "" {
			continue
		}

		r.expireActiveJob(snap, reason, now)
		expired++
	}

	return expired
}

// expireActiveJob fails the tracked job and tells its worker to stop it.
// Failing the record alone leaves the worker running the job and its slot
// taken, which is what kept a stuck job's slot out of reach for days.
func (r *Plugin) expireActiveJob(snap activeJobSnapshot, reason string, now time.Time) {
	r.handleJobCompleted(&plugin_pb.JobCompleted{
		JobId:        snap.jobID,
		JobType:      snap.jobType,
		RequestId:    snap.requestID,
		Success:      false,
		ErrorMessage: reason,
		CompletedAt:  timestamppb.New(now),
	})
	// A retry reuses the job id, so the request id is what names this attempt.
	target := snap.requestID
	if target == "" {
		target = snap.jobID
	}
	if err := r.sendCancel(snap.workerID, target, plugin_pb.WorkKind_WORK_KIND_EXECUTION, errors.New(reason)); err != nil {
		r.holdUndeliveredCancel(snap.jobID, undeliveredCancel{workerID: snap.workerID, target: target, reason: reason})
	}
}

// undeliveredCancel is an expiry cancel its worker never received: it was
// disconnected, or its outgoing buffer stayed full past the send timeout.
type undeliveredCancel struct {
	workerID string
	target   string
	reason   string
}

// holdUndeliveredCancel keeps the cancel for the worker's next heartbeat.
// Entries whose job is no longer tracked are dropped here, so a worker that
// never comes back does not keep them forever.
func (r *Plugin) holdUndeliveredCancel(jobID string, pending undeliveredCancel) {
	r.undeliveredCancelsMu.Lock()
	defer r.undeliveredCancelsMu.Unlock()
	if r.undeliveredCancels == nil {
		r.undeliveredCancels = make(map[string]undeliveredCancel)
	}
	r.jobsMu.RLock()
	for id := range r.undeliveredCancels {
		if _, tracked := r.jobs[id]; !tracked {
			delete(r.undeliveredCancels, id)
		}
	}
	r.jobsMu.RUnlock()
	r.undeliveredCancels[jobID] = pending
}

// resendUndeliveredCancels sends a held cancel again while the worker's
// heartbeat still lists the job, and forgets it otherwise. A cancel that
// can only name the job id is skipped once the job is active again, since
// it would stop a new attempt too.
func (r *Plugin) resendUndeliveredCancels(workerID string, heartbeat *plugin_pb.WorkerHeartbeat) {
	r.undeliveredCancelsMu.Lock()
	var held map[string]undeliveredCancel
	for jobID, pending := range r.undeliveredCancels {
		if pending.workerID != workerID {
			continue
		}
		delete(r.undeliveredCancels, jobID)
		if held == nil {
			held = make(map[string]undeliveredCancel)
		}
		held[jobID] = pending
	}
	r.undeliveredCancelsMu.Unlock()
	if len(held) == 0 {
		return
	}

	for _, work := range heartbeat.GetRunningWork() {
		jobID := strings.TrimSpace(work.GetWorkId())
		pending, ok := held[jobID]
		if !ok || work.GetKind() != plugin_pb.WorkKind_WORK_KIND_EXECUTION {
			continue
		}
		delete(held, jobID)
		if pending.target == jobID {
			r.jobsMu.RLock()
			tracked := r.jobs[jobID]
			active := tracked == nil || isActiveTrackedJobState(tracked.State)
			r.jobsMu.RUnlock()
			if active {
				continue
			}
		}
		go func() {
			if err := r.sendCancel(workerID, pending.target, plugin_pb.WorkKind_WORK_KIND_EXECUTION, errors.New(pending.reason)); err != nil {
				r.holdUndeliveredCancel(jobID, pending)
			}
		}()
	}
}

func (r *Plugin) snapshotActiveJobs() []activeJobSnapshot {
	r.jobsMu.RLock()
	defer r.jobsMu.RUnlock()

	if len(r.jobs) == 0 {
		return nil
	}

	out := make([]activeJobSnapshot, 0, len(r.jobs))
	for _, job := range r.jobs {
		if job == nil {
			continue
		}
		if !isActiveTrackedJobState(job.State) {
			continue
		}
		out = append(out, activeJobSnapshot{
			jobID:      job.JobID,
			jobType:    job.JobType,
			workerID:   job.WorkerID,
			requestID:  job.RequestID,
			lastUpdate: jobLastUpdated(job),
		})
	}
	return out
}

// attemptEnded reports whether the job's record already closed the attempt
// requestID names, as an expiry does while the worker is still running it.
// That attempt's late reports must not reopen the record or log a second run.
func attemptEnded(job *TrackedJob, requestID string) bool {
	return job != nil && requestID != "" && job.RequestID == requestID && !isActiveTrackedJobState(job.State)
}

func jobLastUpdated(job *TrackedJob) time.Time {
	if job == nil {
		return time.Time{}
	}
	if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
		return *job.UpdatedAt
	}
	if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
		return *job.CreatedAt
	}
	return time.Time{}
}

func (r *Plugin) evaluateStaleJob(now time.Time, workerID string, lastUpdate time.Time) (bool, time.Duration, string) {
	if lastUpdate.IsZero() {
		return false, 0, ""
	}

	timeout := defaultStaleActiveJobTimeout
	reason := fmt.Sprintf("job expired after %s without progress", timeout)

	workerID = strings.TrimSpace(workerID)
	if workerID == "" {
		reason = fmt.Sprintf("job expired after %s without executor assignment", timeout)
	} else if !r.isWorkerAvailable(workerID) {
		timeout = defaultOrphanedActiveJobTimeout
		reason = fmt.Sprintf("job expired after %s without worker heartbeat (worker=%s)", timeout, workerID)
	}

	if now.Sub(lastUpdate) < timeout {
		return false, timeout, reason
	}
	return true, timeout, reason
}

func (r *Plugin) confirmStaleReason(now time.Time, jobID string) string {
	r.jobsMu.RLock()
	job := r.jobs[jobID]
	if job == nil || !isActiveTrackedJobState(job.State) {
		r.jobsMu.RUnlock()
		return ""
	}
	lastUpdate := jobLastUpdated(job)
	workerID := job.WorkerID
	r.jobsMu.RUnlock()

	stale, _, reason := r.evaluateStaleJob(now, workerID, lastUpdate)
	if !stale {
		return ""
	}
	return reason
}

func (r *Plugin) isWorkerAvailable(workerID string) bool {
	workerID = strings.TrimSpace(workerID)
	if workerID == "" {
		return false
	}
	_, ok := r.registry.Get(workerID)
	return ok
}

func isTerminalTrackedJobState(state string) bool {
	normalized := strings.ToLower(strings.TrimSpace(state))
	switch normalized {
	case StateSucceeded, StateFailed, StateCanceled:
		return true
	default:
		return false
	}
}

func mergeTerminalDetailIntoTracked(tracked *TrackedJob, detail *TrackedJob) {
	if tracked == nil || detail == nil {
		return
	}
	if !isTerminalTrackedJobState(detail.State) {
		return
	}
	if !isActiveTrackedJobState(tracked.State) {
		return
	}

	if detail.State != "" {
		tracked.State = detail.State
	}
	if detail.Progress != 0 {
		tracked.Progress = detail.Progress
	}
	if detail.Stage != "" {
		tracked.Stage = detail.Stage
	}
	if detail.Message != "" {
		tracked.Message = detail.Message
	}
	if detail.ErrorMessage != "" {
		tracked.ErrorMessage = detail.ErrorMessage
	}
	if detail.ResultSummary != "" {
		tracked.ResultSummary = detail.ResultSummary
	}
	if detail.CompletedAt != nil && !detail.CompletedAt.IsZero() {
		tracked.CompletedAt = detail.CompletedAt
	}
	if detail.UpdatedAt != nil && !detail.UpdatedAt.IsZero() {
		tracked.UpdatedAt = detail.UpdatedAt
	}
	if tracked.UpdatedAt == nil && tracked.CompletedAt != nil {
		tracked.UpdatedAt = tracked.CompletedAt
	}
}

func (r *Plugin) ListTrackedJobs(jobType string, state string, limit int) []TrackedJob {
	r.jobsMu.RLock()
	defer r.jobsMu.RUnlock()

	normalizedJobType := strings.TrimSpace(jobType)
	normalizedState := strings.TrimSpace(strings.ToLower(state))

	items := make([]TrackedJob, 0, len(r.jobs))
	for _, job := range r.jobs {
		if job == nil {
			continue
		}
		if normalizedJobType != "" && job.JobType != normalizedJobType {
			continue
		}
		if normalizedState != "" && strings.ToLower(job.State) != normalizedState {
			continue
		}
		items = append(items, cloneTrackedJob(*job))
	}

	sort.Slice(items, func(i, j int) bool {
		ti := time.Time{}
		if items[i].UpdatedAt != nil {
			ti = *items[i].UpdatedAt
		}
		tj := time.Time{}
		if items[j].UpdatedAt != nil {
			tj = *items[j].UpdatedAt
		}
		if !ti.Equal(tj) {
			return ti.After(tj)
		}
		return items[i].JobID < items[j].JobID
	})

	if limit > 0 && len(items) > limit {
		items = items[:limit]
	}
	return items
}

func (r *Plugin) GetTrackedJob(jobID string) (*TrackedJob, bool) {
	r.jobsMu.RLock()
	defer r.jobsMu.RUnlock()

	job, ok := r.jobs[jobID]
	if !ok || job == nil {
		return nil, false
	}
	clone := cloneTrackedJob(*job)
	return &clone, true
}

func (r *Plugin) ListActivities(jobType string, limit int) []JobActivity {
	r.activitiesMu.RLock()
	defer r.activitiesMu.RUnlock()

	normalized := strings.TrimSpace(jobType)
	activities := make([]JobActivity, 0, len(r.activities))
	for _, activity := range r.activities {
		if normalized != "" && activity.JobType != normalized {
			continue
		}
		activities = append(activities, activity)
	}

	sort.Slice(activities, func(i, j int) bool {
		return activityLess(activities[i], activities[j])
	})
	if limit > 0 && len(activities) > limit {
		activities = activities[:limit]
	}
	return activities
}

func (r *Plugin) ListJobActivities(jobID string, limit int) []JobActivity {
	normalizedJobID := strings.TrimSpace(jobID)
	if normalizedJobID == "" {
		return nil
	}

	r.activitiesMu.RLock()
	activities := make([]JobActivity, 0, len(r.activities))
	for _, activity := range r.activities {
		if strings.TrimSpace(activity.JobID) != normalizedJobID {
			continue
		}
		activities = append(activities, activity)
	}
	r.activitiesMu.RUnlock()

	sort.Slice(activities, func(i, j int) bool {
		return !activityLess(activities[i], activities[j]) // oldest-first for job timeline
	})
	if limit > 0 && len(activities) > limit {
		activities = activities[len(activities)-limit:]
	}
	return activities
}

func (r *Plugin) BuildJobDetail(jobID string, activityLimit int, relatedLimit int) (*JobDetail, bool, error) {
	normalizedJobID := strings.TrimSpace(jobID)
	if normalizedJobID == "" {
		return nil, false, nil
	}

	// Clamp relatedLimit to a safe range to avoid excessive memory allocation from untrusted input.
	if relatedLimit <= 0 {
		relatedLimit = 0
	} else if relatedLimit > maxRelatedJobs {
		relatedLimit = maxRelatedJobs
	}

	r.jobsMu.RLock()
	trackedSnapshot, ok := r.jobs[normalizedJobID]
	if ok && trackedSnapshot != nil {
		candidate := cloneTrackedJob(*trackedSnapshot)
		stripTrackedJobDetailFields(&candidate)
		trackedSnapshot = &candidate
	} else {
		trackedSnapshot = nil
	}
	r.jobsMu.RUnlock()

	detailJob, err := r.store.LoadJobDetail(normalizedJobID)
	if err != nil {
		return nil, false, err
	}

	if trackedSnapshot == nil && detailJob == nil {
		return nil, false, nil
	}
	if detailJob == nil && trackedSnapshot != nil {
		clone := cloneTrackedJob(*trackedSnapshot)
		detailJob = &clone
	}
	if detailJob == nil {
		return nil, false, nil
	}
	if trackedSnapshot != nil {
		mergeTrackedStatusIntoDetail(detailJob, trackedSnapshot)
	}
	detailJob.Parameters = enrichTrackedJobParameters(detailJob.JobType, detailJob.Parameters)

	r.activitiesMu.RLock()
	activities := append([]JobActivity(nil), r.activities...)
	r.activitiesMu.RUnlock()

	detail := &JobDetail{
		Job:         detailJob,
		Activities:  filterJobActivitiesFromSlice(activities, normalizedJobID, activityLimit),
		LastUpdated: timeToPtr(time.Now().UTC()),
	}

	if history, err := r.store.LoadRunHistory(detailJob.JobType); err != nil {
		return nil, true, err
	} else if history != nil {
		for i := range history.SuccessfulRuns {
			record := history.SuccessfulRuns[i]
			if strings.TrimSpace(record.JobID) == normalizedJobID {
				recordCopy := record
				detail.RunRecord = &recordCopy
				break
			}
		}
		if detail.RunRecord == nil {
			for i := range history.ErrorRuns {
				record := history.ErrorRuns[i]
				if strings.TrimSpace(record.JobID) == normalizedJobID {
					recordCopy := record
					detail.RunRecord = &recordCopy
					break
				}
			}
		}
	}

	if relatedLimit > 0 {
		related := make([]TrackedJob, 0, relatedLimit)
		r.jobsMu.RLock()
		for _, candidate := range r.jobs {
			if strings.TrimSpace(candidate.JobType) != strings.TrimSpace(detailJob.JobType) {
				continue
			}
			if strings.TrimSpace(candidate.JobID) == normalizedJobID {
				continue
			}
			cloned := cloneTrackedJob(*candidate)
			stripTrackedJobDetailFields(&cloned)
			related = append(related, cloned)
			if len(related) >= relatedLimit {
				break
			}
		}
		r.jobsMu.RUnlock()
		detail.RelatedJobs = related
	}

	return detail, true, nil
}

func filterJobActivitiesFromSlice(all []JobActivity, jobID string, limit int) []JobActivity {
	if strings.TrimSpace(jobID) == "" || len(all) == 0 {
		return nil
	}

	activities := make([]JobActivity, 0, len(all))
	for _, activity := range all {
		if strings.TrimSpace(activity.JobID) != jobID {
			continue
		}
		activities = append(activities, activity)
	}

	sort.Slice(activities, func(i, j int) bool {
		return !activityLess(activities[i], activities[j]) // oldest-first for job timeline
	})
	if limit > 0 && len(activities) > limit {
		activities = activities[len(activities)-limit:]
	}
	return activities
}

func stripTrackedJobDetailFields(job *TrackedJob) {
	if job == nil {
		return
	}
	job.Detail = ""
	job.Parameters = nil
	job.Labels = nil
	job.ResultOutputValues = nil
}

func hasTrackedJobRichDetails(job TrackedJob) bool {
	return strings.TrimSpace(job.Detail) != "" ||
		len(job.Parameters) > 0 ||
		len(job.Labels) > 0 ||
		len(job.ResultOutputValues) > 0
}

func mergeTrackedStatusIntoDetail(detail *TrackedJob, tracked *TrackedJob) {
	if detail == nil || tracked == nil {
		return
	}

	if detail.JobType == "" {
		detail.JobType = tracked.JobType
	}
	if detail.RequestID == "" {
		detail.RequestID = tracked.RequestID
	}
	if detail.WorkerID == "" {
		detail.WorkerID = tracked.WorkerID
	}
	if detail.DedupeKey == "" {
		detail.DedupeKey = tracked.DedupeKey
	}
	if detail.Summary == "" {
		detail.Summary = tracked.Summary
	}
	if detail.State == "" {
		detail.State = tracked.State
	}
	if detail.Progress == 0 {
		detail.Progress = tracked.Progress
	}
	if detail.Stage == "" {
		detail.Stage = tracked.Stage
	}
	if detail.Message == "" {
		detail.Message = tracked.Message
	}
	if detail.Attempt == 0 {
		detail.Attempt = tracked.Attempt
	}
	if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
		detail.CreatedAt = tracked.CreatedAt
	}
	if detail.UpdatedAt == nil || detail.UpdatedAt.IsZero() {
		detail.UpdatedAt = tracked.UpdatedAt
	}
	if detail.CompletedAt == nil || detail.CompletedAt.IsZero() {
		detail.CompletedAt = tracked.CompletedAt
	}
	if detail.ErrorMessage == "" {
		detail.ErrorMessage = tracked.ErrorMessage
	}
	if detail.ResultSummary == "" {
		detail.ResultSummary = tracked.ResultSummary
	}
}

func (r *Plugin) handleJobProgressUpdate(workerID string, update *plugin_pb.JobProgressUpdate) {
	if update == nil {
		return
	}

	now := time.Now().UTC()
	resolvedWorkerID := strings.TrimSpace(workerID)

	if strings.TrimSpace(update.JobId) != "" {
		r.jobsMu.Lock()
		job := r.jobs[update.JobId]
		if attemptEnded(job, update.RequestId) {
			r.jobsMu.Unlock()
			return
		}
		if job == nil {
			job = &TrackedJob{
				JobID:     update.JobId,
				JobType:   update.JobType,
				RequestID: update.RequestId,
				WorkerID:  resolvedWorkerID,
				CreatedAt: timeToPtr(now),
			}
			r.jobs[update.JobId] = job
		}

		if update.JobType != "" {
			job.JobType = update.JobType
		}
		if update.RequestId != "" {
			job.RequestID = update.RequestId
		}
		if job.WorkerID != "" {
			resolvedWorkerID = job.WorkerID
		} else if resolvedWorkerID != "" {
			job.WorkerID = resolvedWorkerID
		}
		job.State = strings.ToLower(update.State.String())
		job.Progress = update.ProgressPercent
		job.Stage = update.Stage
		job.Message = update.Message
		job.UpdatedAt = timeToPtr(now)
		r.pruneTrackedJobsLocked()
		r.dirtyJobs = true
		r.jobsMu.Unlock()
	}

	r.trackWorkerActivities(update.JobType, update.JobId, update.RequestId, resolvedWorkerID, update.Activities)
	if update.Message != "" || update.Stage != "" {
		source := "worker_progress"
		if strings.TrimSpace(update.JobId) == "" {
			source = "worker_detection"
		}
		r.appendActivity(JobActivity{
			JobID:      update.JobId,
			JobType:    update.JobType,
			RequestID:  update.RequestId,
			WorkerID:   resolvedWorkerID,
			Source:     source,
			Message:    update.Message,
			Stage:      update.Stage,
			OccurredAt: timeToPtr(now),
		})
	}
}

func (r *Plugin) trackExecutionStart(requestID, workerID string, job *plugin_pb.JobSpec, attempt int32) {
	if job == nil || strings.TrimSpace(job.JobId) == "" {
		return
	}

	now := time.Now().UTC()

	r.jobsMu.Lock()
	tracked := r.jobs[job.JobId]
	if tracked == nil {
		tracked = &TrackedJob{
			JobID:     job.JobId,
			CreatedAt: timeToPtr(now),
		}
		r.jobs[job.JobId] = tracked
	}

	tracked.JobType = job.JobType
	tracked.RequestID = requestID
	tracked.WorkerID = workerID
	tracked.DedupeKey = job.DedupeKey
	tracked.Summary = job.Summary
	tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_ASSIGNED.String())
	tracked.Progress = 0
	tracked.Stage = "assigned"
	tracked.Message = "job assigned to worker"
	tracked.Attempt = attempt
	if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
		tracked.CreatedAt = timeToPtr(now)
	}
	tracked.UpdatedAt = timeToPtr(now)
	trackedSnapshot := cloneTrackedJob(*tracked)
	r.pruneTrackedJobsLocked()
	r.dirtyJobs = true
	r.jobsMu.Unlock()
	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
		detail.JobID = job.JobId
		detail.JobType = job.JobType
		detail.RequestID = requestID
		detail.WorkerID = workerID
		detail.DedupeKey = job.DedupeKey
		detail.Summary = job.Summary
		detail.Detail = job.Detail
		detail.Parameters = enrichTrackedJobParameters(job.JobType, configValueMapToPlain(job.Parameters))
		if len(job.Labels) > 0 {
			labels := make(map[string]string, len(job.Labels))
			for key, value := range job.Labels {
				labels[key] = value
			}
			detail.Labels = labels
		} else {
			detail.Labels = nil
		}
		detail.State = trackedSnapshot.State
		detail.Progress = trackedSnapshot.Progress
		detail.Stage = trackedSnapshot.Stage
		detail.Message = trackedSnapshot.Message
		detail.Attempt = attempt
		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
			detail.CreatedAt = trackedSnapshot.CreatedAt
		}
		detail.UpdatedAt = trackedSnapshot.UpdatedAt
	})

	r.appendActivity(JobActivity{
		JobID:      job.JobId,
		JobType:    job.JobType,
		RequestID:  requestID,
		WorkerID:   workerID,
		Source:     "admin_dispatch",
		Message:    "job assigned",
		Stage:      "assigned",
		OccurredAt: timeToPtr(now),
	})
}

func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
	if job == nil || strings.TrimSpace(job.JobId) == "" {
		return
	}

	now := time.Now().UTC()

	r.jobsMu.Lock()
	tracked := r.jobs[job.JobId]
	if tracked == nil {
		tracked = &TrackedJob{
			JobID:     job.JobId,
			CreatedAt: timeToPtr(now),
		}
		r.jobs[job.JobId] = tracked
	}

	tracked.JobType = job.JobType
	tracked.DedupeKey = job.DedupeKey
	tracked.Summary = job.Summary
	tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_PENDING.String())
	tracked.Progress = 0
	tracked.Stage = "queued"
	tracked.Message = "waiting for available executor"
	if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
		tracked.CreatedAt = timeToPtr(now)
	}
	tracked.UpdatedAt = timeToPtr(now)
	trackedSnapshot := cloneTrackedJob(*tracked)
	r.pruneTrackedJobsLocked()
	r.dirtyJobs = true
	r.jobsMu.Unlock()
	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
		detail.JobID = job.JobId
		detail.JobType = job.JobType
		detail.DedupeKey = job.DedupeKey
		detail.Summary = job.Summary
		detail.Detail = job.Detail
		detail.Parameters = enrichTrackedJobParameters(job.JobType, configValueMapToPlain(job.Parameters))
		if len(job.Labels) > 0 {
			labels := make(map[string]string, len(job.Labels))
			for key, value := range job.Labels {
				labels[key] = value
			}
			detail.Labels = labels
		} else {
			detail.Labels = nil
		}
		detail.State = trackedSnapshot.State
		detail.Progress = trackedSnapshot.Progress
		detail.Stage = trackedSnapshot.Stage
		detail.Message = trackedSnapshot.Message
		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
			detail.CreatedAt = trackedSnapshot.CreatedAt
		}
		detail.UpdatedAt = trackedSnapshot.UpdatedAt
	})

	r.appendActivity(JobActivity{
		JobID:      job.JobId,
		JobType:    job.JobType,
		Source:     "admin_scheduler",
		Message:    "job queued for execution",
		Stage:      "queued",
		OccurredAt: timeToPtr(now),
	})
}

// bindQueuedJobWorker records the home worker while the job stays pending/queued.
func (r *Plugin) bindQueuedJobWorker(job *plugin_pb.JobSpec, workerID string) {
	if job == nil || strings.TrimSpace(job.JobId) == "" || strings.TrimSpace(workerID) == "" {
		return
	}

	now := time.Now().UTC()
	r.jobsMu.Lock()
	tracked := r.jobs[job.JobId]
	if tracked == nil {
		r.jobsMu.Unlock()
		return
	}
	if isTerminalTrackedJobState(tracked.State) {
		r.jobsMu.Unlock()
		return
	}
	tracked.WorkerID = workerID
	tracked.Message = "queued on worker " + workerID
	tracked.UpdatedAt = timeToPtr(now)
	snap := cloneTrackedJob(*tracked)
	r.dirtyJobs = true
	r.jobsMu.Unlock()

	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
		detail.WorkerID = snap.WorkerID
		detail.Message = snap.Message
		detail.UpdatedAt = snap.UpdatedAt
	})
}

func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
	reason := "job canceled"
	if cause != nil {
		reason = cause.Error()
	}
	r.markJobCanceled(job, reason)
}

func (r *Plugin) markJobCanceled(job *plugin_pb.JobSpec, reason string) {
	if job == nil || strings.TrimSpace(job.JobId) == "" {
		return
	}

	now := time.Now().UTC()
	if strings.TrimSpace(reason) == "" {
		reason = "job canceled"
	}

	r.jobsMu.Lock()
	tracked := r.jobs[job.JobId]
	if tracked == nil {
		tracked = &TrackedJob{
			JobID:     job.JobId,
			CreatedAt: timeToPtr(now),
		}
		r.jobs[job.JobId] = tracked
	}

	if job.JobType != "" {
		tracked.JobType = job.JobType
	}
	tracked.State = StateCanceled
	tracked.Stage = "canceled"
	tracked.Message = reason
	tracked.ErrorMessage = reason
	tracked.Progress = 0
	if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
		tracked.CreatedAt = timeToPtr(now)
	}
	tracked.UpdatedAt = timeToPtr(now)
	tracked.CompletedAt = timeToPtr(now)
	trackedSnapshot := cloneTrackedJob(*tracked)
	r.pruneTrackedJobsLocked()
	r.dirtyJobs = true
	r.jobsMu.Unlock()

	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
		detail.JobID = job.JobId
		if job.JobType != "" {
			detail.JobType = job.JobType
		}
		detail.State = trackedSnapshot.State
		detail.Stage = trackedSnapshot.Stage
		detail.Message = trackedSnapshot.Message
		detail.ErrorMessage = trackedSnapshot.ErrorMessage
		detail.Progress = trackedSnapshot.Progress
		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
			detail.CreatedAt = trackedSnapshot.CreatedAt
		}
		detail.UpdatedAt = trackedSnapshot.UpdatedAt
		detail.CompletedAt = trackedSnapshot.CompletedAt
	})

	r.appendActivity(JobActivity{
		JobID:      job.JobId,
		JobType:    job.JobType,
		Source:     "admin_scheduler",
		Message:    reason,
		Stage:      "canceled",
		OccurredAt: timeToPtr(now),
	})
}

func (r *Plugin) trackExecutionCompletion(completed *plugin_pb.JobCompleted) *TrackedJob {
	if completed == nil || strings.TrimSpace(completed.JobId) == "" {
		return nil
	}

	now := time.Now().UTC()
	if completed.CompletedAt != nil {
		now = completed.CompletedAt.AsTime().UTC()
	}

	r.jobsMu.Lock()
	tracked := r.jobs[completed.JobId]
	if tracked == nil {
		tracked = &TrackedJob{
			JobID:     completed.JobId,
			CreatedAt: timeToPtr(now),
		}
		r.jobs[completed.JobId] = tracked
	}

	if completed.JobType != "" {
		tracked.JobType = completed.JobType
	}
	if completed.RequestId != "" {
		tracked.RequestID = completed.RequestId
	}
	if completed.Success {
		tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_SUCCEEDED.String())
		tracked.Progress = 100
		tracked.Stage = "completed"
		if completed.Result != nil {
			tracked.ResultSummary = completed.Result.Summary
		}
		tracked.Message = tracked.ResultSummary
		if tracked.Message == "" {
			tracked.Message = "completed"
		}
		tracked.ErrorMessage = ""
	} else {
		tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_FAILED.String())
		tracked.Stage = "failed"
		tracked.ErrorMessage = completed.ErrorMessage
		tracked.Message = completed.ErrorMessage
	}

	tracked.UpdatedAt = timeToPtr(now)
	tracked.CompletedAt = timeToPtr(now)
	r.pruneTrackedJobsLocked()
	clone := cloneTrackedJob(*tracked)
	r.dirtyJobs = true
	r.jobsMu.Unlock()
	r.persistJobDetailSnapshot(completed.JobId, func(detail *TrackedJob) {
		detail.JobID = completed.JobId
		if completed.JobType != "" {
			detail.JobType = completed.JobType
		}
		if completed.RequestId != "" {
			detail.RequestID = completed.RequestId
		}
		detail.State = clone.State
		detail.Progress = clone.Progress
		detail.Stage = clone.Stage
		detail.Message = clone.Message
		detail.ErrorMessage = clone.ErrorMessage
		detail.ResultSummary = clone.ResultSummary
		if completed.Success && completed.Result != nil {
			detail.ResultOutputValues = configValueMapToPlain(completed.Result.OutputValues)
		} else {
			detail.ResultOutputValues = nil
		}
		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
			detail.CreatedAt = clone.CreatedAt
		}
		if detail.UpdatedAt == nil || detail.UpdatedAt.IsZero() {
			detail.UpdatedAt = clone.UpdatedAt
		}
		if detail.CompletedAt == nil || detail.CompletedAt.IsZero() {
			detail.CompletedAt = clone.CompletedAt
		}
	})

	r.appendActivity(JobActivity{
		JobID:      completed.JobId,
		JobType:    completed.JobType,
		RequestID:  completed.RequestId,
		WorkerID:   clone.WorkerID,
		Source:     "worker_completion",
		Message:    clone.Message,
		Stage:      clone.Stage,
		OccurredAt: timeToPtr(now),
	})

	return &clone
}

func (r *Plugin) trackWorkerActivities(jobType, jobID, requestID, workerID string, events []*plugin_pb.ActivityEvent) {
	if len(events) == 0 {
		return
	}
	for _, event := range events {
		if event == nil {
			continue
		}
		timestamp := time.Now().UTC()
		if event.CreatedAt != nil {
			timestamp = event.CreatedAt.AsTime().UTC()
		}
		r.appendActivity(JobActivity{
			JobID:      jobID,
			JobType:    jobType,
			RequestID:  requestID,
			WorkerID:   workerID,
			Source:     strings.ToLower(event.Source.String()),
			Message:    event.Message,
			Stage:      event.Stage,
			Details:    configValueMapToPlain(event.Details),
			OccurredAt: timeToPtr(timestamp),
		})
	}
}

func (r *Plugin) appendActivity(activity JobActivity) {
	if activity.OccurredAt == nil || activity.OccurredAt.IsZero() {
		activity.OccurredAt = timeToPtr(time.Now().UTC())
	}

	r.activitiesMu.Lock()
	r.activities = append(r.activities, activity)
	if len(r.activities) > maxActivityRecords {
		r.activities = r.activities[len(r.activities)-maxActivityRecords:]
	}
	r.dirtyActivities = true
	r.activitiesMu.Unlock()
}

func (r *Plugin) pruneTrackedJobsLocked() {
	if len(r.jobs) <= maxTrackedJobsTotal {
		return
	}

	type sortableJob struct {
		jobID     string
		updatedAt time.Time
	}
	terminalJobs := make([]sortableJob, 0)
	for jobID, job := range r.jobs {
		if job.State == StateSucceeded ||
			job.State == StateFailed ||
			job.State == StateCanceled {
			updAt := time.Time{}
			if job.UpdatedAt != nil {
				updAt = *job.UpdatedAt
			}
			terminalJobs = append(terminalJobs, sortableJob{jobID, updAt})
		}
	}

	if len(terminalJobs) == 0 {
		return
	}

	sort.Slice(terminalJobs, func(i, j int) bool {
		return terminalJobs[i].updatedAt.Before(terminalJobs[j].updatedAt)
	})

	toDelete := len(r.jobs) - maxTrackedJobsTotal
	if toDelete <= 0 {
		return
	}
	if toDelete > len(terminalJobs) {
		toDelete = len(terminalJobs)
	}

	for i := 0; i < toDelete; i++ {
		jobID := terminalJobs[i].jobID
		delete(r.jobs, jobID)
		if err := r.store.DeleteJobDetail(jobID); err != nil {
			glog.V(2).Infof("Plugin failed to delete job detail for pruned job %s: %v", jobID, err)
		}
	}
}

func configValueMapToPlain(values map[string]*plugin_pb.ConfigValue) map[string]interface{} {
	if len(values) == 0 {
		return nil
	}

	payload, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(&plugin_pb.ValueMap{Fields: values})
	if err != nil {
		return nil
	}

	decoded := map[string]interface{}{}
	if err := json.Unmarshal(payload, &decoded); err != nil {
		return nil
	}

	fields, ok := decoded["fields"].(map[string]interface{})
	if !ok {
		return nil
	}
	return fields
}

func (r *Plugin) persistTrackedJobsSnapshot() {
	r.jobsMu.Lock()
	r.dirtyJobs = false
	jobs := make([]TrackedJob, 0, len(r.jobs))
	for _, job := range r.jobs {
		if job == nil || strings.TrimSpace(job.JobID) == "" {
			continue
		}
		clone := cloneTrackedJob(*job)
		stripTrackedJobDetailFields(&clone)
		jobs = append(jobs, clone)
	}
	r.jobsMu.Unlock()

	if len(jobs) == 0 {
		return
	}

	sort.Slice(jobs, func(i, j int) bool {
		ti := time.Time{}
		if jobs[i].UpdatedAt != nil {
			ti = *jobs[i].UpdatedAt
		}
		tj := time.Time{}
		if jobs[j].UpdatedAt != nil {
			tj = *jobs[j].UpdatedAt
		}
		if !ti.Equal(tj) {
			return ti.After(tj)
		}
		return jobs[i].JobID < jobs[j].JobID
	})
	if len(jobs) > maxTrackedJobsTotal {
		jobs = jobs[:maxTrackedJobsTotal]
	}

	if err := r.store.SaveTrackedJobs(jobs); err != nil {
		glog.Warningf("Plugin failed to persist tracked jobs: %v", err)
	}
}

func (r *Plugin) persistJobDetailSnapshot(jobID string, apply func(detail *TrackedJob)) {
	normalizedJobID, _ := sanitizeJobID(jobID)
	if normalizedJobID == "" {
		return
	}

	r.jobDetailsMu.Lock()
	defer r.jobDetailsMu.Unlock()

	detail, err := r.store.LoadJobDetail(normalizedJobID)
	if err != nil {
		glog.Warningf("Plugin failed to load job detail snapshot for %s: %v", normalizedJobID, err)
		return
	}
	if detail == nil {
		detail = &TrackedJob{
			JobID: normalizedJobID,
		}
	}

	if apply != nil {
		apply(detail)
	}

	if err := r.store.SaveJobDetail(*detail); err != nil {
		glog.Warningf("Plugin failed to persist job detail snapshot for %s: %v", normalizedJobID, err)
	}
}

func (r *Plugin) persistActivitiesSnapshot() {
	r.activitiesMu.Lock()
	r.dirtyActivities = false
	activities := append([]JobActivity(nil), r.activities...)
	r.activitiesMu.Unlock()

	if len(activities) == 0 {
		return
	}

	if len(activities) > maxActivityRecords {
		activities = activities[len(activities)-maxActivityRecords:]
	}

	if err := r.store.SaveActivities(activities); err != nil {
		glog.Warningf("Plugin failed to persist activities: %v", err)
	}
}

func (r *Plugin) persistenceLoop() {
	defer r.wg.Done()
	for {
		select {
		case <-r.shutdownCh:
			r.persistTrackedJobsSnapshot()
			r.persistActivitiesSnapshot()
			return
		case <-r.persistTicker.C:
			r.jobsMu.RLock()
			needsJobsFlush := r.dirtyJobs
			r.jobsMu.RUnlock()
			if needsJobsFlush {
				r.persistTrackedJobsSnapshot()
			}

			r.activitiesMu.RLock()
			needsActivitiesFlush := r.dirtyActivities
			r.activitiesMu.RUnlock()
			if needsActivitiesFlush {
				r.persistActivitiesSnapshot()
			}
		}
	}
}