The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(orchestration): extract background completion lifecycle helpers
Steven Enamakelee15ecfe5d13
diff --git a/crates/openhuman-core/src/agent/orchestration/background_completions.rs b/crates/openhuman-core/src/agent/orchestration/background_completions.rsindex c420374b92..3e07cfecf9 100644--- a/crates/openhuman-core/src/agent/orchestration/background_completions.rs+++ b/crates/openhuman-core/src/agent/orchestration/background_completions.rs@@ -518,237 +518,6 @@ fn mark_collected_in_entry(entry: Arc<Entry>, task_id: &str) -> bool {matches!(completion, Ok(TombstoneOutcome::Suppressed)) || pauses_collected}-/// Drop every queued completion for `thread_id` and everything that finishes-/// for it later. Called when the thread is deleted: the router's deleted-parent-/// marker is durable, and the thread is also remembered as deleted in memory, so-/// a straggler that wins the cooperative-abort race, a child that registers after-/// the delete, or a result that finishes after a restart is dropped rather than-/// delivered into a thread that no longer exists. Takes the caller's workspace so-/// the marker is written even when no child of the thread was seen by this-/// process. Returns the number of queued completions removed.-pub(crate) fn discard_for_thread(- ctx: &DispatchCtx,- workspace_dir: &Path,- thread_id: &str,-) -> usize {- let seen = {- let thread_key = key(ctx, thread_id);- let mut st = state();- st.deleted_threads.insert(thread_key.clone());- st.stopped_threads.remove(&thread_key);- st.thread_workspaces.contains_key(&thread_key)- };- let entry = entry_for_ctx(ctx, workspace_dir);- // An ordinary thread that never admitted background work needs only the- // in-memory fence. Durable lifecycle markers and settled completion history- // both remain in the store after compaction, so a previously used thread- // still gets a permanent delete marker after a restart.- let has_history = match entry.store.try_list(Some(thread_id)) {- Ok(records) => !records.is_empty(),- Err(error) => {- log::warn!(- "[background_completions] could not inspect thread history before delete \- thread_id={thread_id} error={error}"- );- true- }- };- let has_legacy_delete_marker = is_marked_deleted(&entry, thread_id);- if !seen && !has_history && !has_legacy_delete_marker {- return 0;- }- let removed = cancel_deleted_parent(&entry, thread_id);- log::debug!(- "[background_completions] discard_for_thread thread_id={thread_id} removed={removed}"- );- removed-}--/// Task id of the durable "this thread was deleted" marker. The router's-/// cancelled-parent marker cannot tell a delete from a Stop (a Stop is lifted-/// when the user returns), so a delete also leaves this one, which nothing-/// lifts. It is stored `Pending` (with no parent, so nothing ever claims it):-/// compaction only drops settled records, and a deletion must outlive them.-fn deleted_marker_id(thread_id: &str) -> String {- format!("\u{1}host-thread-deleted:{thread_id}")-}--fn is_marked_deleted(entry: &Entry, thread_id: &str) -> bool {- entry.store.get(&deleted_marker_id(thread_id)).is_some()-}--/// Permanently delete `thread_id` in the router. Failures are logged; the-/// in-memory deleted set still gates this process.-fn cancel_deleted_parent(entry: &Entry, thread_id: &str) -> usize {- entry- .router- .delete_parent(thread_id)- .unwrap_or_else(|error| {- log::error!(- "[background_completions] delete_parent failed thread_id={thread_id} error={error}"- );- 0- })-}--/// Drop every queued completion for `thread_id` and gate late results from the-/// stopped generation.-///-/// The Stop-button counterpart of [`discard_for_thread`]: the user halted the-/// thread's work, so results that finished but were not yet delivered must not-/// start a fresh delivery turn behind their back. The thread stays alive; the-/// gate lifts at [`resume_stopped_thread`] (the next accepted user turn), while-/// the stopped generation's task ids stay tombstoned-/// ([`finish_stop_for_thread`]). Returns the number of queued completions removed.-pub(crate) fn discard_pending_for_thread(ctx: &DispatchCtx, thread_id: &str) -> usize {- state().stopped_threads.insert(key(ctx, thread_id));- let Some(router) = router_for_thread(ctx, thread_id) else {- return 0;- };- let removed = router.cancel_parent(thread_id).unwrap_or_else(|error| {- log::error!(- "[background_completions] cancel_parent failed thread_id={thread_id} error={error}"- );- 0- });- log::debug!(- "[background_completions] discard_pending_for_thread thread_id={thread_id} removed={removed}"- );- removed-}--/// Complete a Stop after its registered children were aborted: tombstone their-/// task ids so a straggler from the stopped generation cannot record once a-/// later user turn reopens the thread.-pub(crate) fn finish_stop_for_thread(ctx: &DispatchCtx, thread_id: &str, task_ids: &[String]) {- let Some(router) = router_for_thread(ctx, thread_id) else {- return;- };- for task_id in task_ids {- if let Err(error) = router.tombstone(task_id) {- log::warn!("[background_completions] tombstone failed task_id={task_id} error={error}");- }- }-}--/// Reopen a thread's completion gate for a newly accepted user turn.-///-/// The Stop gate deliberately outlives registry cancellation, because a detached-/// child may be between `tokio::spawn` and `running_subagents::register` when-/// Stop is pressed. New task ids remain distinct from the stopped generation.-/// Only a thread this process stopped or cancelled pays for the durable resume-/// marker, so an ordinary chat message writes nothing.-pub(crate) fn resume_stopped_thread(ctx: &DispatchCtx, thread_id: &str) {- let was_stopped = state().stopped_threads.remove(&key(ctx, thread_id));- if !was_stopped {- return;- }- if let Some(router) = router_for_thread(ctx, thread_id) {- router.resume_parent(thread_id);- log::debug!("[background_completions] resumed thread_id={thread_id}");- }-}--/// Record a child that registers while its parent thread is stopped or deleted.-///-/// Registration happens after the detached task is spawned. If Stop (or delete)-/// races that narrow interval the registry sweep cannot see the child;-/// tombstoning its task id here keeps it rejected even after a later user turn-/// reopens a stopped thread. Also notes which workspace holds the thread's-/// completions, so a later thread-scoped Stop can find the router. The first-/// child this process spawns on a (not deleted) thread lifts any-/// cancelled-parent marker an earlier process left behind (a Stop before a-/// restart): a live spawn proves the user re-engaged the thread. Returns whether-/// the child must be aborted.-pub(crate) fn mark_stopped_task_if_thread_stopped(- ctx: &DispatchCtx,- workspace_dir: &Path,- thread_id: &str,- task_id: &str,-) -> bool {- let entry = entry_for_ctx(ctx, workspace_dir);- let thread_key = key(ctx, thread_id);- let (first_sight, mut stopped) = {- let mut st = state();- let first_sight = st- .thread_workspaces- .insert(thread_key.clone(), workspace_dir.to_path_buf())- .is_none();- let stopped =- st.stopped_threads.contains(&thread_key) || st.deleted_threads.contains(&thread_key);- (first_sight, stopped)- };- // Existing host delete markers predate the router's distinct permanent- // deletion state. Preserve them, then consult the owner's lifecycle marker- // for threads deleted by current versions.- if first_sight && !stopped {- if is_marked_deleted(&entry, thread_id) {- state().deleted_threads.insert(thread_key.clone());- stopped = true;- } else {- // A Stop is explicitly lifted by a new accepted child, but a- // deleted parent cannot be resumed. Re-admit after attempting the- // resume so deletion and storage errors fail closed.- entry.router.resume_parent(thread_id);- match entry.router.admit_child_generation(thread_id) {- Ok(Some(_)) => {}- admission => {- // Neither a closed lifecycle nor a transient store error- // proves permanent deletion: resume may have failed while- // the subsequent admission successfully read the old Stop- // marker. Refuse this child and let the next one retry.- if let Err(error) = admission {- log::warn!(- "[background_completions] parent admission failed thread_id={thread_id} error={error}"- );- }- if let Err(tombstone_error) = entry.router.tombstone(task_id) {- log::warn!("[background_completions] tombstone failed task_id={task_id} error={tombstone_error}");- }- let mut st = state();- if st- .thread_workspaces- .get(&thread_key)- .is_some_and(|path| path == workspace_dir)- {- st.thread_workspaces.remove(&thread_key);- }- return true;- }- }- }- }- if stopped {- if let Err(error) = entry.router.tombstone(task_id) {- log::warn!("[background_completions] tombstone failed task_id={task_id} error={error}");- }- return true;- }- false-}--/// Withdraw every queued completion of `workspace_dir`. Called on a full thread-/// purge of that workspace; each thread with undelivered results is cancelled-/// durably (and remembered as deleted), so stragglers are still dropped. Other-/// workspaces are untouched. Returns the number of completions removed.-pub(crate) fn clear_all(ctx: &DispatchCtx, workspace_dir: &Path) -> usize {- let entry = entry_for_ctx(ctx, workspace_dir);- let parents: HashSet<String> = entry- .store- .list(None)- .into_iter()- .filter(|r| r.state == CompletionState::Pending && !r.parent_key.is_empty())- .map(|r| r.parent_key)- .collect();- let mut removed = 0;- for parent in parents {- state().deleted_threads.insert(key(ctx, &parent));- removed += cancel_deleted_parent(&entry, &parent);- }- log::debug!("[background_completions] clear_all removed={removed}");- removed-}-/// Boot recovery for a workspace: open its log, drop long-settled records, and/// return the threads that still hold undelivered completions (a previous/// process finished the work but never delivered it). Remembers each thread'sdiff --git a/crates/openhuman-core/src/agent/orchestration/background_completions_parent_lifecycle.rs b/crates/openhuman-core/src/agent/orchestration/background_completions_parent_lifecycle.rsnew file mode 100644index 0000000000..35fa6ac8bc--- /dev/null+++ b/crates/openhuman-core/src/agent/orchestration/background_completions_parent_lifecycle.rs@@ -0,0 +1,233 @@+use super::*;++/// Drop every queued completion for `thread_id` and everything that finishes+/// for it later. Called when the thread is deleted: the router's deleted-parent+/// marker is durable, and the thread is also remembered as deleted in memory, so+/// a straggler that wins the cooperative-abort race, a child that registers after+/// the delete, or a result that finishes after a restart is dropped rather than+/// delivered into a thread that no longer exists. Takes the caller's workspace so+/// the marker is written even when no child of the thread was seen by this+/// process. Returns the number of queued completions removed.+pub(crate) fn discard_for_thread(+ ctx: &DispatchCtx,+ workspace_dir: &Path,+ thread_id: &str,+) -> usize {+ let seen = {+ let thread_key = key(ctx, thread_id);+ let mut st = state();+ st.deleted_threads.insert(thread_key.clone());+ st.stopped_threads.remove(&thread_key);+ st.thread_workspaces.contains_key(&thread_key)+ };+ let entry = entry_for_ctx(ctx, workspace_dir);+ // An ordinary thread that never admitted background work needs only the+ // in-memory fence. Durable lifecycle markers and settled completion history+ // both remain in the store after compaction, so a previously used thread+ // still gets a permanent delete marker after a restart.+ let has_history = match entry.store.try_list(Some(thread_id)) {+ Ok(records) => !records.is_empty(),+ Err(error) => {+ log::warn!(+ "[background_completions] could not inspect thread history before delete \+ thread_id={thread_id} error={error}"+ );+ true+ }+ };+ let has_legacy_delete_marker = is_marked_deleted(&entry, thread_id);+ if !seen && !has_history && !has_legacy_delete_marker {+ return 0;+ }+ let removed = cancel_deleted_parent(&entry, thread_id);+ log::debug!(+ "[background_completions] discard_for_thread thread_id={thread_id} removed={removed}"+ );+ removed+}++/// Task id of the durable "this thread was deleted" marker. The router's+/// cancelled-parent marker cannot tell a delete from a Stop (a Stop is lifted+/// when the user returns), so a delete also leaves this one, which nothing+/// lifts. It is stored `Pending` (with no parent, so nothing ever claims it):+/// compaction only drops settled records, and a deletion must outlive them.+fn deleted_marker_id(thread_id: &str) -> String {+ format!("\u{1}host-thread-deleted:{thread_id}")+}++fn is_marked_deleted(entry: &Entry, thread_id: &str) -> bool {+ entry.store.get(&deleted_marker_id(thread_id)).is_some()+}++/// Permanently delete `thread_id` in the router. Failures are logged; the+/// in-memory deleted set still gates this process.+fn cancel_deleted_parent(entry: &Entry, thread_id: &str) -> usize {+ entry+ .router+ .delete_parent(thread_id)+ .unwrap_or_else(|error| {+ log::error!(+ "[background_completions] delete_parent failed thread_id={thread_id} error={error}"+ );+ 0+ })+}++/// Drop every queued completion for `thread_id` and gate late results from the+/// stopped generation.+///+/// The Stop-button counterpart of [`discard_for_thread`]: the user halted the+/// thread's work, so results that finished but were not yet delivered must not+/// start a fresh delivery turn behind their back. The thread stays alive; the+/// gate lifts at [`resume_stopped_thread`] (the next accepted user turn), while+/// the stopped generation's task ids stay tombstoned+/// ([`finish_stop_for_thread`]). Returns the number of queued completions removed.+pub(crate) fn discard_pending_for_thread(ctx: &DispatchCtx, thread_id: &str) -> usize {+ state().stopped_threads.insert(key(ctx, thread_id));+ let Some(router) = router_for_thread(ctx, thread_id) else {+ return 0;+ };+ let removed = router.cancel_parent(thread_id).unwrap_or_else(|error| {+ log::error!(+ "[background_completions] cancel_parent failed thread_id={thread_id} error={error}"+ );+ 0+ });+ log::debug!(+ "[background_completions] discard_pending_for_thread thread_id={thread_id} removed={removed}"+ );+ removed+}++/// Complete a Stop after its registered children were aborted: tombstone their+/// task ids so a straggler from the stopped generation cannot record once a+/// later user turn reopens the thread.+pub(crate) fn finish_stop_for_thread(ctx: &DispatchCtx, thread_id: &str, task_ids: &[String]) {+ let Some(router) = router_for_thread(ctx, thread_id) else {+ return;+ };+ for task_id in task_ids {+ if let Err(error) = router.tombstone(task_id) {+ log::warn!("[background_completions] tombstone failed task_id={task_id} error={error}");+ }+ }+}++/// Reopen a thread's completion gate for a newly accepted user turn.+///+/// The Stop gate deliberately outlives registry cancellation, because a detached+/// child may be between `tokio::spawn` and `running_subagents::register` when+/// Stop is pressed. New task ids remain distinct from the stopped generation.+/// Only a thread this process stopped or cancelled pays for the durable resume+/// marker, so an ordinary chat message writes nothing.+pub(crate) fn resume_stopped_thread(ctx: &DispatchCtx, thread_id: &str) {+ let was_stopped = state().stopped_threads.remove(&key(ctx, thread_id));+ if !was_stopped {+ return;+ }+ if let Some(router) = router_for_thread(ctx, thread_id) {+ router.resume_parent(thread_id);+ log::debug!("[background_completions] resumed thread_id={thread_id}");+ }+}++/// Record a child that registers while its parent thread is stopped or deleted.+///+/// Registration happens after the detached task is spawned. If Stop (or delete)+/// races that narrow interval the registry sweep cannot see the child;+/// tombstoning its task id here keeps it rejected even after a later user turn+/// reopens a stopped thread. Also notes which workspace holds the thread's+/// completions, so a later thread-scoped Stop can find the router. The first+/// child this process spawns on a (not deleted) thread lifts any+/// cancelled-parent marker an earlier process left behind (a Stop before a+/// restart): a live spawn proves the user re-engaged the thread. Returns whether+/// the child must be aborted.+pub(crate) fn mark_stopped_task_if_thread_stopped(+ ctx: &DispatchCtx,+ workspace_dir: &Path,+ thread_id: &str,+ task_id: &str,+) -> bool {+ let entry = entry_for_ctx(ctx, workspace_dir);+ let thread_key = key(ctx, thread_id);+ let (first_sight, mut stopped) = {+ let mut st = state();+ let first_sight = st+ .thread_workspaces+ .insert(thread_key.clone(), workspace_dir.to_path_buf())+ .is_none();+ let stopped =+ st.stopped_threads.contains(&thread_key) || st.deleted_threads.contains(&thread_key);+ (first_sight, stopped)+ };+ // Existing host delete markers predate the router's distinct permanent+ // deletion state. Preserve them, then consult the owner's lifecycle marker+ // for threads deleted by current versions.+ if first_sight && !stopped {+ if is_marked_deleted(&entry, thread_id) {+ state().deleted_threads.insert(thread_key.clone());+ stopped = true;+ } else {+ // A Stop is explicitly lifted by a new accepted child, but a+ // deleted parent cannot be resumed. Re-admit after attempting the+ // resume so deletion and storage errors fail closed.+ entry.router.resume_parent(thread_id);+ match entry.router.admit_child_generation(thread_id) {+ Ok(Some(_)) => {}+ admission => {+ // Neither a closed lifecycle nor a transient store error+ // proves permanent deletion: resume may have failed while+ // the subsequent admission successfully read the old Stop+ // marker. Refuse this child and let the next one retry.+ if let Err(error) = admission {+ log::warn!(+ "[background_completions] parent admission failed thread_id={thread_id} error={error}"+ );+ }+ if let Err(tombstone_error) = entry.router.tombstone(task_id) {+ log::warn!("[background_completions] tombstone failed task_id={task_id} error={tombstone_error}");+ }+ let mut st = state();+ if st+ .thread_workspaces+ .get(&thread_key)+ .is_some_and(|path| path == workspace_dir)+ {+ st.thread_workspaces.remove(&thread_key);+ }+ return true;+ }+ }+ }+ }+ if stopped {+ if let Err(error) = entry.router.tombstone(task_id) {+ log::warn!("[background_completions] tombstone failed task_id={task_id} error={error}");+ }+ return true;+ }+ false+}++/// Withdraw every queued completion of `workspace_dir`. Called on a full thread+/// purge of that workspace; each thread with undelivered results is cancelled+/// durably (and remembered as deleted), so stragglers are still dropped. Other+/// workspaces are untouched. Returns the number of completions removed.+pub(crate) fn clear_all(ctx: &DispatchCtx, workspace_dir: &Path) -> usize {+ let entry = entry_for_ctx(ctx, workspace_dir);+ let parents: HashSet<String> = entry+ .store+ .list(None)+ .into_iter()+ .filter(|r| r.state == CompletionState::Pending && !r.parent_key.is_empty())+ .map(|r| r.parent_key)+ .collect();+ let mut removed = 0;+ for parent in parents {+ state().deleted_threads.insert(key(ctx, &parent));+ removed += cancel_deleted_parent(&entry, &parent);+ }+ log::debug!("[background_completions] clear_all removed={removed}");+ removed+}+