Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: signal blocked scheduler workers on every newly-dequeueable state change

Luigi Agosti735916806fc2
diff --git a/internal/scheduler/queue.go b/internal/scheduler/queue.go
index ddbd0d38..a6f76b59 100644
--- a/internal/scheduler/queue.go
+++ b/internal/scheduler/queue.go
@@ -31,6 +31,16 @@ func NewTaskQueue(maxTotal, maxPerAgent int) *TaskQueue {
}
}
+// signal wakes one worker blocked on Ready. notify holds a single pending
+// wake-up, so every path that leaves work dequeueable must call this; workers
+// have no poll fallback. Callers must not hold q.mu.
+func (q *TaskQueue) signal() {
+ select {
+ case q.notify <- struct{}{}:
+ default:
+ }
+}
+
// SetLimits updates queue capacity at runtime.
func (q *TaskQueue) SetLimits(maxTotal, maxPerAgent int) {
q.mu.Lock()
@@ -69,11 +79,7 @@ func (q *TaskQueue) Enqueue(t *Task) (int, error) {
pos := q.totalCount
q.mu.Unlock()
- // Wake a blocked worker.
- select {
- case q.notify <- struct{}{}:
- default:
- }
+ q.signal()
return pos, nil
}
@@ -82,6 +88,16 @@ func (q *TaskQueue) Enqueue(t *Task) (int, error) {
// fewest in-flight tasks gets served first. Among tasks for that agent,
// the heap ordering (priority then creation time) decides.
func (q *TaskQueue) Dequeue(maxPerAgentInflight, maxGlobalInflight int) *Task {
+ t, remaining := q.dequeue(maxPerAgentInflight, maxGlobalInflight)
+ if remaining {
+ q.signal()
+ }
+ return t
+}
+
+// dequeue pops the next eligible task and reports whether queued work is left
+// behind, so the caller can pass the single pending wake-up along.
+func (q *TaskQueue) dequeue(maxPerAgentInflight, maxGlobalInflight int) (*Task, bool) {
q.mu.Lock()
defer q.mu.Unlock()
@@ -90,7 +106,7 @@ func (q *TaskQueue) Dequeue(maxPerAgentInflight, maxGlobalInflight int) *Task {
globalInflight += aq.inflight
}
if globalInflight >= maxGlobalInflight {
- return nil
+ return nil, false
}
var bestAgent string
@@ -110,36 +126,34 @@ func (q *TaskQueue) Dequeue(maxPerAgentInflight, maxGlobalInflight int) *Task {
}
if bestAgent == "" {
- return nil
+ return nil, false
}
aq := q.agents[bestAgent]
t := heap.Pop(&aq.tasks).(*Task)
aq.inflight++
q.totalCount--
- return t
+ return t, q.totalCount > 0
}
// Complete marks a task as no longer in-flight for its agent.
func (q *TaskQueue) Complete(agentID string) {
q.mu.Lock()
- hasQueued := false
if aq, ok := q.agents[agentID]; ok {
if aq.inflight > 0 {
aq.inflight--
}
- hasQueued = aq.tasks.Len() > 0
- if aq.inflight == 0 && !hasQueued {
+ if aq.inflight == 0 && aq.tasks.Len() == 0 {
delete(q.agents, agentID)
}
}
+ // Any queued task may now be eligible: this completion frees a global
+ // inflight slot, not just one for agentID.
+ hasQueued := q.totalCount > 0
q.mu.Unlock()
if hasQueued {
- select {
- case q.notify <- struct{}{}:
- default:
- }
+ q.signal()
}
}
diff --git a/internal/scheduler/queue_wakeup_test.go b/internal/scheduler/queue_wakeup_test.go
new file mode 100644
index 00000000..4e891d7b
--- /dev/null
+++ b/internal/scheduler/queue_wakeup_test.go
@@ -0,0 +1,70 @@
+package scheduler
+
+import "testing"
+
+func drainReady(q *TaskQueue) {
+ select {
+ case <-q.Ready():
+ default:
+ }
+}
+
+// Workers block on Ready() with no poll fallback, so every state change that
+// can newly permit a Dequeue must signal. Completing a task frees a global
+// inflight slot that another agent's queued task needs.
+func TestCompleteSignalsWhenOtherAgentHasQueuedWork(t *testing.T) {
+ q := NewTaskQueue(10, 10)
+
+ if _, err := q.Enqueue(&Task{ID: "a1", AgentID: "A"}); err != nil {
+ t.Fatalf("enqueue a1: %v", err)
+ }
+ drainReady(q)
+
+ running := q.Dequeue(5, 1)
+ if running == nil {
+ t.Fatal("a1 should dequeue")
+ }
+
+ if _, err := q.Enqueue(&Task{ID: "b1", AgentID: "B"}); err != nil {
+ t.Fatalf("enqueue b1: %v", err)
+ }
+ drainReady(q)
+
+ if blocked := q.Dequeue(5, 1); blocked != nil {
+ t.Fatalf("b1 must be blocked by the global inflight cap, got %s", blocked.ID)
+ }
+
+ q.Complete(running.AgentID)
+
+ select {
+ case <-q.Ready():
+ default:
+ t.Fatal("Complete freed the only global inflight slot but did not signal; b1 stalls until an unrelated Enqueue")
+ }
+ if got := q.Dequeue(5, 1); got == nil || got.ID != "b1" {
+ t.Fatalf("b1 should dequeue after the slot frees, got %v", got)
+ }
+}
+
+// notify holds a single pending signal, so a burst of enqueues wakes only one
+// worker. A successful Dequeue that leaves work behind must hand the wake-up on.
+func TestDequeueCascadesWakeupWhenWorkRemains(t *testing.T) {
+ q := NewTaskQueue(10, 10)
+
+ for _, id := range []string{"a1", "a2", "a3"} {
+ if _, err := q.Enqueue(&Task{ID: id, AgentID: "A"}); err != nil {
+ t.Fatalf("enqueue %s: %v", id, err)
+ }
+ }
+ drainReady(q)
+
+ if got := q.Dequeue(5, 5); got == nil {
+ t.Fatal("first task should dequeue")
+ }
+
+ select {
+ case <-q.Ready():
+ default:
+ t.Fatal("Dequeue left queued work but did not signal; idle workers stay blocked")
+ }
+}
diff --git a/internal/scheduler/reload.go b/internal/scheduler/reload.go
index f87114a7..41ffc9b3 100644
--- a/internal/scheduler/reload.go
+++ b/internal/scheduler/reload.go
@@ -21,6 +21,9 @@ func (s *Scheduler) ReloadConfig(cfg Config) {
s.cfg.MaxInflight = cfg.MaxInflight
s.cfg.MaxPerAgentFlight = cfg.MaxPerAgentFlight
s.cfgMu.Unlock()
+ // A raised cap can make queued tasks eligible; workers parked on Ready
+ // have no poll fallback and would otherwise wait for unrelated traffic.
+ s.queue.signal()
slog.Info("scheduler: inflight limits reloaded",
"maxInflight", cfg.MaxInflight,
"maxPerAgentFlight", cfg.MaxPerAgentFlight,