Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows
Show what's pending when `wt list` stalls >5s (#2203)
Maximilian Roos309194520d97
diff --git a/src/commands/list/collect/mod.rs b/src/commands/list/collect/mod.rsindex 1c617d148..81c1a69a3 100644--- a/src/commands/list/collect/mod.rs+++ b/src/commands/list/collect/mod.rs@@ -881,10 +881,10 @@ pub fn collect(// Drain task results with conditional progressive rendering.//// Progressive mutable state (table, row cache, counters) is owned by a- // `RefCell` so the `on_result` callback and the one-shot 200ms tick can- // both mutate it. They never run concurrently — the tick fires between- // channel recvs — so the runtime borrow checks are an invariant formalism,- // never a source of panics.+ // `RefCell` so the event callback (handling results, the one-shot 200ms+ // reveal, and stall hints) can mutate it. Events never run concurrently —+ // they fire between channel recvs — so the runtime borrow checks are an+ // invariant formalism, never a source of panics.struct ProgressiveState {table: ProgressiveTable,last_rendered_lines: Vec<String>,@@ -907,30 +907,7 @@ pub fn collect(let drain_deadline =collect_deadline.unwrap_or_else(|| std::time::Instant::now() + results::DRAIN_TIMEOUT);- // Tick closure: at T+200ms, promote placeholder from blank to `·` and- // re-render every row so still-pending cells pick up the dot. Rows that- // have already received a result use `format_list_item_line`; untouched- // rows use `render_skeleton_row` (which avoids surfacing seeded defaults- // like "55y" for skipped tasks).- let tick_entry: Option<results::DrainTick<'_>> = progressive_state.as_ref().map(|state_cell| {- let f: results::DrainTickFn<'_> = Box::new(|items: &mut [ListItem]| {- layout.placeholder.set(super::render::PLACEHOLDER);- let mut s = state_cell.borrow_mut();- for (idx, item) in items.iter().enumerate() {- let rendered = if s.last_rendered_lines[idx].is_empty() {- layout.render_skeleton_row(item).render()- } else {- layout.format_list_item_line(item)- };- if rendered != s.last_rendered_lines[idx] {- s.last_rendered_lines[idx] = rendered.clone();- s.table.update_row(idx, rendered);- }- }- let _ = s.table.flush();- });- (placeholder_reveal_at, f)- });+ let reveal_at = progressive_state.as_ref().map(|_| placeholder_reveal_at);let drain_outcome = drain_results(rx,@@ -939,46 +916,92 @@ pub fn collect(&expected_results,drain_deadline,integration_target.as_deref(),- |item_idx, item| {+ |event| {let Some(state_cell) = progressive_state.as_ref() else {return;};let mut s = state_cell.borrow_mut();-- if !s.first_result_traced {- s.first_result_traced = true;- worktrunk::shell_exec::trace_instant("First result received");- }-let dim = Style::new().dimmed();- s.completed_results += 1;let total_results = expected_results.count();- debug_assert!(- s.completed_results <= total_results,- "completed ({}) > expected ({}): task result sent without registering expectation",- s.completed_results,- total_results- );- if s.completed_results > total_results {- s.progress_overflow = true;- }-- let completed = s.completed_results;- let footer_msg = format!(- "{INFO_SYMBOL} {dim}{footer_base} ({completed}/{total_results} loaded){dim:#}"- );- s.table.update_footer(footer_msg);-- let rendered = layout.format_list_item_line(item);- if rendered != s.last_rendered_lines[item_idx] {- s.last_rendered_lines[item_idx] = rendered.clone();- s.table.update_row(item_idx, rendered);+ match event {+ results::DrainEvent::Result { item_idx, item } => {+ if !s.first_result_traced {+ s.first_result_traced = true;+ worktrunk::shell_exec::trace_instant("First result received");+ }++ s.completed_results += 1;++ debug_assert!(+ s.completed_results <= total_results,+ "completed ({}) > expected ({}): task result sent without registering expectation",+ s.completed_results,+ total_results+ );+ if s.completed_results > total_results {+ s.progress_overflow = true;+ }++ let completed = s.completed_results;+ let footer_msg = format!(+ "{INFO_SYMBOL} {dim}{footer_base} ({completed}/{total_results} loaded){dim:#}"+ );+ s.table.update_footer(footer_msg);++ let rendered = layout.format_list_item_line(item);+ if rendered != s.last_rendered_lines[item_idx] {+ s.last_rendered_lines[item_idx] = rendered.clone();+ s.table.update_row(item_idx, rendered);+ }++ if let Err(e) = s.table.flush() {+ log::debug!("Progressive table flush failed: {}", e);+ }+ }+ results::DrainEvent::Reveal { items } => {+ // T+200ms reveal: promote blank placeholders to `·` and+ // re-render every row so still-pending cells pick up+ // the dot. Rows that received a result use+ // `format_list_item_line`; untouched rows use+ // `render_skeleton_row` (which avoids surfacing seeded+ // defaults like "55y" for skipped tasks).+ layout.placeholder.set(super::render::PLACEHOLDER);+ for (idx, item) in items.iter().enumerate() {+ let rendered = if s.last_rendered_lines[idx].is_empty() {+ layout.render_skeleton_row(item).render()+ } else {+ layout.format_list_item_line(item)+ };+ if rendered != s.last_rendered_lines[idx] {+ s.last_rendered_lines[idx] = rendered.clone();+ s.table.update_row(idx, rendered);+ }+ }+ if let Err(e) = s.table.flush() {+ log::debug!("Progressive table reveal flush failed: {}", e);+ }+ }+ results::DrainEvent::Stall { pending } => {+ // No task has completed for at least `STALL_TIMINGS.threshold`.+ // Name the signal (silence) rather than claiming "stalled":+ // the event fires on any 5s lull and points at the first+ // pending hint, not a root cause.+ let completed = s.completed_results;+ let kind_str: &'static str = pending.kind.into();+ let footer_msg = cformat!(+ "{INFO_SYMBOL} {dim}{footer_base} ({completed}/{total_results} loaded, no recent progress; waiting on <underline>{kind_str}</> for <underline>{name}</>){dim:#}",+ name = pending.name+ );+ if s.table.update_footer(footer_msg)+ && let Err(e) = s.table.flush()+ {+ log::debug!("Progressive table flush failed: {}", e);+ }+ }}-- let _ = s.table.flush();},- tick_entry,+ reveal_at,);worktrunk::shell_exec::trace_instant("All results drained");@@ -992,7 +1015,7 @@ pub fn collect(None => (None, false),};// Reveal the placeholder synchronously for any path where the drain- // finished before the tick could fire — keeps subsequent renders+ // finished before the reveal could fire — keeps subsequent renders// (including `finalize`) consistent with the post-reveal placeholder.layout.placeholder.set(super::render::PLACEHOLDER);@@ -1313,7 +1336,7 @@ pub fn populate_item(&expected_results,std::time::Instant::now() + results::DRAIN_TIMEOUT,target.as_deref(),- |_item_idx, _item| {},+ |_event| {},None,);diff --git a/src/commands/list/collect/results.rs b/src/commands/list/collect/results.rsindex 4eccbaff2..eef04ef9b 100644--- a/src/commands/list/collect/results.rs+++ b/src/commands/list/collect/results.rs@@ -16,22 +16,78 @@ use std::time::{Duration, Instant};use crossbeam_channel as chan;+use super::super::model::{ItemKind, ListItem};+use super::execution::ExpectedResults;+use super::types::{DrainOutcome, MissingResult, TaskError, TaskKind, TaskResult};+/// Deadline for the entire drain operation. Generous to avoid flaky timeouts/// under CI load where process spawning for ~70 work items can be slow.pub(super) const DRAIN_TIMEOUT: Duration = Duration::from_secs(120);-use super::super::model::{ItemKind, ListItem};-use super::execution::ExpectedResults;-use super::types::{DrainOutcome, MissingResult, TaskError, TaskKind, TaskResult};+/// Stall detection timings used by production callers of `drain_results`.+pub(super) const STALL_TIMINGS: StallTimings = StallTimings {+ threshold: Duration::from_secs(5),+ tick: Duration::from_millis(500),+};++/// Timings controlling `DrainEvent::Stall` emission.+///+/// `threshold` is how long the drain waits with no incoming result before+/// emitting a stall event. `tick` is how often the drain wakes up while idle+/// to re-check — lower values surface the hint faster at the cost of a few+/// extra syscalls per second.+#[derive(Debug, Clone, Copy)]+pub(super) struct StallTimings {+ pub threshold: Duration,+ pub tick: Duration,+}-/// Boxed one-shot tick closure. Factored out so the inferred closure can-/// coerce to `dyn FnMut` at the `Box::new` site.-pub(super) type DrainTickFn<'a> = Box<dyn FnMut(&mut [ListItem]) + 'a>;+/// A single task the drain is still waiting on — surfaced to the caller when+/// no results have arrived in the last `STALL_TIMINGS.threshold`.+#[derive(Debug, Clone, Copy)]+pub(super) struct PendingHint<'a> {+ pub kind: TaskKind,+ pub name: &'a str,+}-/// One-shot tick scheduled against [`drain_results`]. When `Instant` elapses,-/// the boxed closure fires once with the full item slice — used by `wt list`-/// to reveal the `·` loading indicator at T+200ms.-pub(super) type DrainTick<'a> = (Instant, DrainTickFn<'a>);+/// Events emitted by `drain_results`. A single callback handles every+/// event kind so the caller can share state (e.g. the progressive table)+/// across them without fighting the borrow checker.+pub(super) enum DrainEvent<'a> {+ /// A task result has been applied to `item` — progressive mode re-renders.+ Result {+ item_idx: usize,+ item: &'a mut ListItem,+ },+ /// The `reveal_at` deadline passed — fires exactly once. `wt list` uses+ /// this to promote blank placeholders to the `·` loading indicator.+ Reveal { items: &'a [ListItem] },+ /// No results for at least `STALL_TIMINGS.threshold`; here's one task+ /// we're still waiting on. Fires repeatedly (each `STALL_TIMINGS.tick`)+ /// while stalled.+ Stall { pending: PendingHint<'a> },+}++/// Return the first pending `(item_idx, kind)` pair — an expected result+/// that has not yet been received. Iteration order mirrors how work items+/// were registered, giving a deterministic pick when multiple are pending.+fn pick_pending_hint<'a>(+ expected: &ExpectedResults,+ received_by_item: &[Vec<TaskKind>],+ items: &'a [ListItem],+) -> Option<PendingHint<'a>> {+ for ((item_idx, received), item) in received_by_item.iter().enumerate().zip(items.iter()) {+ for kind in expected.results_for(item_idx) {+ if !received.contains(&kind) {+ return Some(PendingHint {+ kind,+ name: item.display_name(),+ });+ }+ }+ }+ None+}/// Drain task results from the channel and apply them to items.///@@ -58,30 +114,63 @@ pub(super) fn drain_results(expected_results: &ExpectedResults,deadline: Instant,integration_target: Option<&str>,- mut on_result: impl FnMut(usize, &mut ListItem),- mut tick: Option<DrainTick<'_>>,+ on_event: impl FnMut(DrainEvent<'_>),+ reveal_at: Option<Instant>,+) -> DrainOutcome {+ drain_results_with_timings(+ rx,+ items,+ errors,+ expected_results,+ deadline,+ integration_target,+ STALL_TIMINGS,+ on_event,+ reveal_at,+ )+}++/// Core drain loop. Stall timings are parameterized so tests can trigger the+/// stall path quickly; production callers use [`drain_results`].+#[allow(clippy::too_many_arguments)]+pub(super) fn drain_results_with_timings(+ rx: chan::Receiver<Result<TaskResult, TaskError>>,+ items: &mut [ListItem],+ errors: &mut Vec<TaskError>,+ expected_results: &ExpectedResults,+ deadline: Instant,+ integration_target: Option<&str>,+ stall_timings: StallTimings,+ mut on_event: impl FnMut(DrainEvent<'_>),+ mut reveal_at: Option<Instant>,) -> DrainOutcome {// Track which result kinds we've received per item (for timeout diagnostics)let mut received_by_item: Vec<Vec<TaskKind>> = vec![Vec::new(); items.len()];+ // Last time a result arrived (start with now so stall is measured from+ // the beginning of collection, not from the Unix epoch).+ let mut last_result_time = Instant::now();+// Process task results as they arrive (with deadline)loop {- // Fire the one-shot tick when its deadline has passed. The tick fires- // between channel recvs, so it never races with `on_result`.- if let Some((tick_at, _)) = tick.as_ref()- && Instant::now() >= *tick_at+ // Fire the one-shot reveal when its deadline has passed. Reveal fires+ // between channel recvs, so it never races with `DrainEvent::Result`.+ if let Some(at) = reveal_at+ && Instant::now() >= at{- let (_, mut tick_fn) = tick.take().unwrap();- tick_fn(items);+ on_event(DrainEvent::Reveal { items });+ reveal_at = None;}let now = Instant::now();let remaining = deadline.saturating_duration_since(now);- // Clamp recv timeout so we wake at the tick deadline (earliest of the two).- let recv_timeout_dur = match tick.as_ref() {- Some((tick_at, _)) => remaining.min(tick_at.saturating_duration_since(now)),- None => remaining,- };+ // Clamp recv timeout to the nearest of: real deadline, stall tick+ // (so we can fire stall events promptly), and the reveal deadline+ // (so we wake to emit it).+ let mut recv_timeout_dur = remaining.min(stall_timings.tick);+ if let Some(at) = reveal_at {+ recv_timeout_dur = recv_timeout_dur.min(at.saturating_duration_since(now));+ }if remaining.is_zero() {// Deadline exceeded - build diagnostic info showing MISSING resultslet received_count: usize = received_by_item.iter().map(|v| v.len()).sum();@@ -104,13 +193,9 @@ pub(super) fn drain_results(.collect();if !missing_kinds.is_empty() {- let name = item- .branch- .clone()- .unwrap_or_else(|| item.head[..8.min(item.head.len())].to_string());items_with_missing.push(MissingResult {item_idx,- name,+ name: item.display_name().to_string(),missing_kinds,});}@@ -128,7 +213,21 @@ pub(super) fn drain_results(let outcome = match rx.recv_timeout(recv_timeout_dur) {Ok(outcome) => outcome,- Err(chan::RecvTimeoutError::Timeout) => continue, // Check tick/deadline next iteration+ Err(chan::RecvTimeoutError::Timeout) => {+ // Nothing arrived within the tick. If we've been silent for+ // at least `stall_timings.threshold`, emit a stall event+ // pointing at one task we're still waiting on. Fires+ // repeatedly while stalled — `update_footer` is idempotent+ // for same content. The loop top handles firing any+ // due one-shot tick and re-checks the real deadline.+ if last_result_time.elapsed() >= stall_timings.threshold+ && let Some(pending) =+ pick_pending_hint(expected_results, &received_by_item, items)+ {+ on_event(DrainEvent::Stall { pending });+ }+ continue;+ }Err(chan::RecvTimeoutError::Disconnected) => break, // All senders dropped - done};@@ -140,6 +239,7 @@ pub(super) fn drain_results(// Track this result for diagnostics (both success and error count as "received")received_by_item[item_idx].push(kind);+ last_result_time = Instant::now();// Errors leave the errored task's fields at `None`. The// corresponding gate stays unresolved (renders `·`). Callers@@ -148,7 +248,10 @@ pub(super) fn drain_results(// footer progress counter advances.if let Err(error) = outcome {errors.push(error);- on_result(item_idx, &mut items[item_idx]);+ on_event(DrainEvent::Result {+ item_idx,+ item: &mut items[item_idx],+ });continue;}@@ -260,7 +363,7 @@ pub(super) fn drain_results(item.refresh_status_symbols(integration_target);// Invoke callback (progressive mode re-renders rows, buffered mode does nothing)- on_result(item_idx, item);+ on_event(DrainEvent::Result { item_idx, item });}DrainOutcome::Complete@@ -365,7 +468,7 @@ mod tests {&expected,Instant::now() + DRAIN_TIMEOUT,None,- |_, _| {},+ |_| {},None,);assert!(matches!(outcome, DrainOutcome::Complete));@@ -444,7 +547,7 @@ mod tests {&expected,Instant::now() + DRAIN_TIMEOUT,Some("main"),- |_, _| {},+ |_| {},None,);@@ -488,7 +591,7 @@ mod tests {&expected,Instant::now() + DRAIN_TIMEOUT,Some("main"),- |_, _| {},+ |_| {},None,);@@ -529,7 +632,7 @@ mod tests {&expected,Instant::now() + DRAIN_TIMEOUT,Some("main"),- |_, _| {},+ |_| {},None,);@@ -560,7 +663,7 @@ mod tests {&expected,Instant::now(),None,- |_, _| {},+ |_| {},None,);@@ -588,18 +691,62 @@ mod tests {}#[test]- fn test_drain_results_tick_fires_when_deadline_passes() {- // A tick whose deadline has already elapsed should fire exactly once- // with a mutable view of the items, then stop (no re-fire on subsequent- // loop iterations). The drain loop wakes at the tick deadline even- // without any channel traffic, so this also exercises the- // recv_timeout-clamping path.- let (tx, rx) = crossbeam_channel::unbounded::<Result<TaskResult, TaskError>>();+ fn test_drain_results_fires_stall_when_silent_past_threshold() {+ // No results arrive; the drain should emit a Stall event pointing at+ // the first pending task once `stall_threshold` elapses, and keep+ // firing on each tick until the deadline.+ let (_tx, rx) = crossbeam_channel::unbounded();+ let mut items = vec![+ ListItem::new_branch("abc123".into(), "feat".into()),+ ListItem::new_branch("def456".into(), "other".into()),+ ];+ let mut errors = Vec::new();++ let expected = ExpectedResults::default();+ expected.expect(0, TaskKind::AheadBehind);+ expected.expect(1, TaskKind::CommitDetails);++ let mut stall_events: Vec<(TaskKind, String)> = Vec::new();+ let outcome = drain_results_with_timings(+ rx,+ &mut items,+ &mut errors,+ &expected,+ Instant::now() + Duration::from_millis(200),+ None,+ StallTimings {+ threshold: Duration::from_millis(20),+ tick: Duration::from_millis(20),+ },+ |event| {+ if let DrainEvent::Stall { pending } = event {+ stall_events.push((pending.kind, pending.name.to_string()));+ }+ },+ None,+ );++ assert!(matches!(outcome, DrainOutcome::TimedOut { .. }));+ assert!(+ !stall_events.is_empty(),+ "expected at least one stall event before the deadline"+ );+ let (kind, name) = &stall_events[0];+ assert_eq!(*kind, TaskKind::AheadBehind);+ assert_eq!(name, "feat");+ }++ #[test]+ fn test_drain_results_does_not_fire_stall_when_results_flow() {+ // Results arrive faster than the stall threshold, so no Stall event+ // should fire. last_result_time is reset on each received result.+ let (tx, rx) = crossbeam_channel::unbounded();let mut items = vec![ListItem::new_branch("abc123".into(), "feat".into())];let mut errors = Vec::new();+let expected = ExpectedResults::default();+ expected.expect(0, TaskKind::CommitDetails);- // Send one result so the drain exits promptly after the tick.tx.send(Ok(TaskResult::SummaryGenerate {item_idx: 0,summary: None,@@ -607,26 +754,81 @@ mod tests {.unwrap();drop(tx);- let mut tick_fires = 0usize;- let tick_fn: DrainTickFn<'_> = Box::new(|items: &mut [ListItem]| {- tick_fires += 1;- // Mutate an item to prove we got a live mutable reference.- items[0].branch = Some("touched-by-tick".into());+ let mut stall_count = 0;+ let outcome = drain_results_with_timings(+ rx,+ &mut items,+ &mut errors,+ &expected,+ Instant::now() + Duration::from_millis(50),+ None,+ StallTimings {+ threshold: Duration::from_secs(10), // far exceeds deadline+ tick: Duration::from_millis(20),+ },+ |event| {+ if matches!(event, DrainEvent::Stall { .. }) {+ stall_count += 1;+ }+ },+ None,+ );++ assert!(matches!(outcome, DrainOutcome::Complete));+ assert_eq!(stall_count, 0, "stall must not fire under the threshold");+ }++ #[test]+ fn test_drain_results_survives_mid_stall_result() {+ // Smoke test: a result arriving mid-stall doesn't crash the loop or+ // double-fire. Does NOT prove `last_result_time` resets — that's+ // visible in the drain source directly. Verifying reset timing in+ // a test would need an upper-bound stall count, and `threshold/tick`+ // is too small a window to distinguish reset-works from reset-broken+ // without flakiness.+ let (tx, rx) = crossbeam_channel::unbounded();+ let mut items = vec![ListItem::new_branch("abc123".into(), "feat".into())];+ let mut errors = Vec::new();++ let expected = ExpectedResults::default();+ expected.expect(0, TaskKind::CommitDetails);+ expected.expect(0, TaskKind::AheadBehind);++ // Send exactly one result partway through.+ std::thread::spawn(move || {+ std::thread::sleep(Duration::from_millis(80));+ let _ = tx.send(Ok(TaskResult::SummaryGenerate {+ item_idx: 0,+ summary: None,+ }));+ // Leave tx open so the drain keeps running until deadline.+ std::thread::sleep(Duration::from_millis(500));+ drop(tx);});- let outcome = drain_results(+ let mut stall_events = 0;+ let _ = drain_results_with_timings(rx,&mut items,&mut errors,&expected,- Instant::now() + DRAIN_TIMEOUT,+ Instant::now() + Duration::from_millis(200),+ None,+ StallTimings {+ threshold: Duration::from_millis(40),+ tick: Duration::from_millis(20),+ },+ |event| {+ if matches!(event, DrainEvent::Stall { .. }) {+ stall_events += 1;+ }+ },None,- |_, _| {},- Some((Instant::now(), tick_fn)),);- assert!(matches!(outcome, DrainOutcome::Complete));- assert_eq!(tick_fires, 1, "tick should fire exactly once");- assert_eq!(items[0].branch.as_deref(), Some("touched-by-tick"));+ assert!(+ stall_events >= 1,+ "drain should still emit stalls even when a result arrives mid-stall"+ );}}diff --git a/src/commands/list/model/item.rs b/src/commands/list/model/item.rsindex a2391027b..6fb184a84 100644--- a/src/commands/list/model/item.rs+++ b/src/commands/list/model/item.rs@@ -331,6 +331,16 @@ impl ListItem {self.branch.as_deref().unwrap_or("(detached)")}+ /// Short display name for this item — the branch if present, otherwise+ /// a truncated HEAD SHA. Use when reporting which item is pending,+ /// stuck, or missing: `branch_name()`'s `"(detached)"` fallback collapses+ /// distinct detached items into one label.+ pub fn display_name(&self) -> &str {+ self.branch+ .as_deref()+ .unwrap_or_else(|| &self.head[..8.min(self.head.len())])+ }+pub fn is_main(&self) -> bool {matches!(&self.kind, ItemKind::Worktree(data) if data.is_main)}