YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(app): recover control-port startup with graceful termination (#7627)
Ezra Ellette9131cfbf29d4
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff.rs b/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff.rsindex bb9956a36..e42f0398f 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff.rs@@ -60,7 +60,7 @@ fn report_failure(root: &std::path::Path, stage: &str, error: &str) {#[cfg(target_os = "macos")]mod macos;#[cfg(target_os = "macos")]-pub(crate) use macos::{initialize, install, owns_launch, pending, reopen};+pub(crate) use macos::{initialize, install, owns_launch, pending, process_stamp, reopen};#[cfg(test)]mod tests {diff --git a/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/macos.rs b/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/macos.rsindex 249c0f032..c712f5cb9 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/macos.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/macos.rs@@ -21,6 +21,7 @@ use tauri::Manager;mod kernel_signature;#[path = "signing.rs"]mod signing;+pub(crate) use signing::process_stamp;const WAIT: Duration = Duration::from_secs(30);const EXPECTED_BUILD_ENV: &str = "SCREENPIPE_MANUAL_HANDOFF_BUILD";diff --git a/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/signing.rs b/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/signing.rsindex 9b9dffa2d..9a2066132 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/signing.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/manual_handoff/signing.rs@@ -85,7 +85,7 @@ pub(super) fn hex(bytes: &[u8]) -> String {bytes.iter().map(|b| format!("{b:02x}")).collect()}-pub(super) fn process_stamp(pid: i32) -> Result<(u32, u64)> {+pub(crate) fn process_stamp(pid: i32) -> Result<(u32, u64)> {let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() };let len = unsafe {libc::proc_pidinfo(diff --git a/apps/screenpipe-app-tauri/src-tauri/src/port_conflict.rs b/apps/screenpipe-app-tauri/src-tauri/src/port_conflict.rsindex afc0fb99e..a4a44dd7a 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/port_conflict.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/port_conflict.rs@@ -55,7 +55,9 @@ pub fn show_healthy_screenpipe(app: &AppHandle, port: u16) {app.dialog().message(message)- .title(crate::localization::ui_text("screenpipe is already running"))+ .title(crate::localization::ui_text(+ "screenpipe is already running",+ )).buttons(MessageDialogButtons::Ok).show(|_| {});}@@ -75,6 +77,84 @@ pub fn show_reclaim_failed(app: &AppHandle, port: u16) {.show(|_| {});}+/// Keep ownership recovery tied to the actual bind result. Address conflicts+/// are transient work for this server task, not a five-attempt startup failure.+pub(crate) async fn bind_control_port<T, E>(+ port: u16,+ healthy_owner: bool,+ root: &std::path::Path,+ mut bind: impl FnMut() -> Result<T, E>,+) -> anyhow::Result<T>+where+ E: std::error::Error + Send + Sync + 'static,+{+ let mut retrying = false;+ loop {+ let error = match bind() {+ Ok(listener) => {+ if retrying {+ crate::update_diagnostics::append(+ root,+ "control_port_recovered",+ &format!("port={port}; outcome=listening"),+ );+ }+ return Ok(listener);+ }+ Err(error) => anyhow::Error::new(error),+ };+ let address_in_use = error.chain().any(|cause| {+ cause+ .downcast_ref::<std::io::Error>()+ .is_some_and(|error| error.kind() == std::io::ErrorKind::AddrInUse)+ });+ if !retrying || !address_in_use {+ crate::update_diagnostics::append(+ root,+ "control_port_bind_failed",+ &format!(+ "port={port}; cause={error:#}; outcome={}",+ if address_in_use {+ "retrying"+ } else {+ "bind_failed"+ }+ ),+ );+ }+ if !address_in_use || healthy_owner {+ return Err(error);+ }+ #[cfg(unix)]+ {+ reclaim_unix_listener(port, LSOF, root).await?;+ }+ #[cfg(windows)]+ reclaim_unhealthy_owner_windows(port).await;+ retrying = true;+ // Reuse the port-release polling cadence. The owning server task+ // cancels this retry when the app shuts down. An empty lsof snapshot+ // is not a permanent failure: the owner may already have exited.+ tokio::time::sleep(std::time::Duration::from_millis(250)).await;+ }+}++pub(crate) fn show_control_start_failed(app: &AppHandle, port: u16, error: &anyhow::Error) {+ if PORT_CONFLICT_REPORTED.swap(true, Ordering::SeqCst) {+ return;+ }+ app.dialog()+ .message(crate::localization::ui_format(+ "local server on port {port}: {error}",+ &[("port", port.to_string()), ("error", format!("{error:#}"))],+ ))+ .title(crate::localization::ui_text(+ "screenpipe could not start its local server",+ ))+ .buttons(MessageDialogButtons::Ok)+ .show(|_| {});+}+pub fn is_error(error: &str, port: u16) -> bool {error.starts_with(&format!("port {port} is already in use"))}@@ -85,48 +165,154 @@ pub async fn reclaim_owner(port: u16, healthy_screenpipe: bool) {}#[cfg(unix)]- reclaim_unhealthy_owner_unix(port).await;+ reclaim_unhealthy_owner_unix(port, crate::config::app_data_dir()).await;#[cfg(windows)]reclaim_unhealthy_owner_windows(port).await;}+// Finder/LaunchServices need not supply /usr/sbin in PATH.+#[cfg(target_os = "macos")]+const LSOF: &str = "/usr/sbin/lsof";+#[cfg(all(unix, not(target_os = "macos")))]+const LSOF: &str = "lsof";+#[cfg(unix)]-async fn reclaim_unhealthy_owner_unix(port: u16) {- let my_pid = std::process::id().to_string();- let output = match tokio::time::timeout(+async fn reclaim_unhealthy_owner_unix(port: u16, root: &std::path::Path) {+ let _ = reclaim_unix_listener(port, LSOF, root).await;+}++#[cfg(unix)]+fn record_reclaim_result(+ root: &std::path::Path,+ port: u16,+ signal: &str,+ result: &anyhow::Result<Vec<i32>>,+) {+ let detail = match result {+ Ok(pids) if pids.is_empty() => return,+ Ok(pids) => {+ format!("port={port}; pids={pids:?}; signal={signal}; outcome=termination_sent")+ }+ Err(error) => format!("port={port}; cause={error:#}; outcome=port_not_reclaimed"),+ };+ tracing::info!("port reclaim: {detail}");+ crate::update_diagnostics::append(root, "port_reclaim", &detail);+}++#[cfg(unix)]+async fn reclaim_unix_listener(+ port: u16,+ lsof: &str,+ root: &std::path::Path,+) -> anyhow::Result<()> {+ let result = async {+ let pids = unix_listener_pids(port, lsof).await?;+ let owners: Vec<_> = pids+ .into_iter()+ .filter_map(|pid| unix_process_generation(pid).map(|generation| (pid, generation)))+ .collect();+ let mut signalled = Vec::new();+ for (pid, generation) in &owners {+ if unix_process_generation(*pid).as_ref() == Some(generation)+ && signal_unix_owner(*pid, port, libc::SIGTERM)?+ {+ signalled.push(*pid);+ }+ }+ record_reclaim_result(root, port, "SIGTERM", &Ok(signalled));+ if owners.is_empty() {+ return Ok(Vec::new());+ }+ // Restore the pre-existing three-second termination grace period.+ wait_for_port_release(port, std::time::Duration::from_secs(3)).await;+ // Discovery must be fresh after waiting. Never escalate against a+ // replacement listener, a reused PID, or an owner that released its port.+ let current_pids = unix_listener_pids(port, lsof).await?;+ let mut forced = Vec::new();+ for (pid, generation) in owners {+ if current_pids.contains(&pid)+ && unix_process_generation(pid).as_ref() == Some(&generation)+ && signal_unix_owner(pid, port, libc::SIGKILL)?+ {+ forced.push(pid);+ }+ }+ Ok(forced)+ }+ .await;+ record_reclaim_result(root, port, "SIGKILL", &result);+ result.map(|_| ())+}++#[cfg(target_os = "macos")]+fn unix_process_generation(pid: i32) -> Option<String> {+ // Reuse the kernel process start stamp used by manual-copy replacement.+ crate::manual_handoff::process_stamp(pid)+ .ok()+ .map(|stamp| format!("{stamp:?}"))+}++#[cfg(all(unix, not(target_os = "macos")))]+fn unix_process_generation(pid: i32) -> Option<String> {+ // /proc stat field 22 is the process start tick. The command field can+ // contain spaces and parentheses, so begin after its final closing ')'.+ let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;+ stat.rsplit_once(')')?+ .1+ .split_whitespace()+ .nth(19)+ .map(str::to_owned)+}++#[cfg(unix)]+fn signal_unix_owner(pid: i32, port: u16, signal: i32) -> anyhow::Result<bool> {+ use anyhow::Context;+ // No PATH-dependent external kill command.+ if unsafe { libc::kill(pid, signal) } == 0 {+ return Ok(true);+ }+ let error = std::io::Error::last_os_error();+ if error.raw_os_error() == Some(libc::ESRCH) {+ return Ok(false);+ }+ Err(error)+ .with_context(|| format!("termination failed; signal={signal}; pid={pid}; port={port}"))+}++#[cfg(unix)]+async fn unix_listener_pids(port: u16, lsof: &str) -> anyhow::Result<Vec<i32>> {+ use anyhow::{bail, Context};+ let output = tokio::time::timeout(std::time::Duration::from_secs(5),- tokio::process::Command::new("lsof")+ tokio::process::Command::new(lsof).args(["-nP", &format!("-tiTCP:{port}"), "-sTCP:LISTEN"])+ .kill_on_drop(true).output(),).await- {- Ok(Ok(output)) if output.status.success() => output,- _ => return,- };-- let output_text = String::from_utf8_lossy(&output.stdout);- let pids: Vec<&str> = output_text- .lines()- .filter(|pid| !pid.is_empty() && *pid != my_pid)- .collect();- for pid in &pids {- let _ = tokio::process::Command::new("kill")- .args(["-TERM", pid])- .output()- .await;+ .with_context(|| format!("listener discovery timed out; command={lsof}; timeout_ms=5000"))?+ .with_context(|| format!("could not inspect port owner; command={lsof}"))?;+ // lsof exits 1 without output when no matching listener exists.+ if output.status.code() == Some(1) && output.stdout.is_empty() && output.stderr.is_empty() {+ return Ok(Vec::new());}- wait_for_port_release(port, std::time::Duration::from_secs(3)).await;- if port_is_available(port).await {- return;+ if !output.status.success() {+ bail!(+ "listener discovery failed; command={lsof}; status={}; stderr={}",+ output.status,+ String::from_utf8_lossy(&output.stderr)+ );}- for pid in &pids {- let _ = tokio::process::Command::new("kill")- .args(["-KILL", pid])- .output()- .await;+ let my_pid = std::process::id() as i32;+ let mut pids = Vec::new();+ for line in std::str::from_utf8(&output.stdout)?.lines() {+ let pid: i32 = line.parse().context("invalid listener PID")?;+ if pid > 0 && pid != my_pid && !pids.contains(&pid) {+ pids.push(pid);+ }}+ Ok(pids)}#[cfg(windows)]@@ -187,6 +373,179 @@ async fn port_is_available(port: u16) -> bool {mod tests {use super::{listening_pid_on_port, should_reclaim_owner};+ #[cfg(target_os = "macos")]+ #[tokio::test]+ async fn unix_reclaim_fixture() {+ let Ok(mode) = std::env::var("SCREENPIPE_PORT_TEST_MODE") else {+ return;+ };+ let root = std::path::PathBuf::from(std::env::var_os("SCREENPIPE_PORT_TEST_ROOT").unwrap());+ if mode != "reclaim" {+ if mode == "owner" {+ unsafe {+ libc::signal(libc::SIGTERM, libc::SIG_IGN);+ }+ }+ let mut terminate = if mode == "release" {+ Some(+ tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())+ .unwrap(),+ )+ } else {+ None+ };+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let ready = root.join("ready");+ std::fs::write(+ root.join("pending"),+ listener.local_addr().unwrap().port().to_string(),+ )+ .unwrap();+ std::fs::rename(root.join("pending"), ready).unwrap();+ if let Some(terminate) = &mut terminate {+ terminate.recv().await;+ drop(listener);+ std::future::pending::<()>().await;+ } else {+ std::future::pending::<()>().await;+ drop(listener);+ }+ } else {+ let port: u16 = std::fs::read_to_string(root.join("ready"))+ .unwrap()+ .parse()+ .unwrap();+ let addr = ([127, 0, 0, 1], port).into();+ let _server =+ super::bind_control_port(port, false, &root, || axum::Server::try_bind(&addr))+ .await+ .unwrap();+ std::fs::write(root.join("recovered"), "listening").unwrap();+ }+ }++ #[cfg(target_os = "macos")]+ #[tokio::test]+ async fn unix_control_port_escalates_only_for_stubborn_owner() {+ assert_unix_reclaim("owner", Some(libc::SIGKILL)).await;+ }++ #[cfg(target_os = "macos")]+ #[tokio::test]+ async fn unix_control_port_uses_normal_termination_first() {+ assert_unix_reclaim("cooperative", Some(libc::SIGTERM)).await;+ }++ #[cfg(target_os = "macos")]+ #[tokio::test]+ async fn unix_control_port_does_not_kill_owner_after_it_releases_port() {+ assert_unix_reclaim("release", None).await;+ }++ #[cfg(target_os = "macos")]+ async fn assert_unix_reclaim(mode: &str, expected_signal: Option<i32>) {+ let root = tempfile::tempdir().unwrap();+ let unrelated = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let fixture = || {+ let mut command = tokio::process::Command::new(std::env::current_exe().unwrap());+ command+ .args(["--exact", "port_conflict::tests::unix_reclaim_fixture"])+ .env("SCREENPIPE_PORT_TEST_ROOT", root.path())+ .kill_on_drop(true);+ command+ };+ let mut owner = fixture()+ .env("SCREENPIPE_PORT_TEST_MODE", mode)+ .spawn()+ .unwrap();+ tokio::time::timeout(std::time::Duration::from_secs(30), async {+ while !root.path().join("ready").exists() {+ assert!(owner.try_wait().unwrap().is_none());+ tokio::time::sleep(std::time::Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap();+ let status = fixture()+ .env("SCREENPIPE_PORT_TEST_MODE", "reclaim")+ .env("PATH", "")+ .status()+ .await+ .unwrap();+ assert!(status.success());+ assert!(root.path().join("recovered").exists());+ use std::os::unix::process::ExitStatusExt;+ if let Some(signal) = expected_signal {+ assert_eq!(owner.wait().await.unwrap().signal(), Some(signal));+ } else {+ assert!(+ owner.try_wait().unwrap().is_none(),+ "released-port owner must survive"+ );+ owner.kill().await.unwrap();+ owner.wait().await.unwrap();+ }+ assert!(std::net::TcpStream::connect(unrelated.local_addr().unwrap()).is_ok());+ let report = crate::diagnostic_logs::collect_redacted_from_dirs(&[root.path().into()])+ .await+ .unwrap();+ assert!(report.contains("signal=SIGTERM"));+ assert_eq!(+ report.contains("signal=SIGKILL"),+ expected_signal == Some(libc::SIGKILL)+ );+ assert!(report.contains("outcome=listening"));+ }++ #[cfg(unix)]+ #[tokio::test]+ async fn vanished_owner_does_not_exhaust_startup_retries() {+ let root = tempfile::tempdir().unwrap();+ let reservation = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = reservation.local_addr().unwrap();+ drop(reservation);+ let mut attempts = 0;+ let _listener = super::bind_control_port(addr.port(), false, root.path(), || {+ attempts += 1;+ // Cross the old five-attempt ceiling with no discoverable owner.+ if attempts <= 6 {+ Err(std::io::Error::from(std::io::ErrorKind::AddrInUse))+ } else {+ std::net::TcpListener::bind(addr)+ }+ })+ .await+ .unwrap();+ assert_eq!(attempts, 7);+ }++ #[tokio::test]+ async fn actual_bind_failure_survives_rotation_in_support_report() {+ let root = tempfile::tempdir().unwrap();+ let error = super::bind_control_port(11435, false, root.path(), || {+ Err::<(), _>(std::io::Error::from_raw_os_error(libc::EMFILE))+ })+ .await+ .unwrap_err();+ assert!(error.to_string().contains("24"));+ for day in 1..=7 {+ std::fs::write(+ root.path()+ .join(format!("screenpipe-app.2026-10-{day:02}.log")),+ "restart\n",+ )+ .unwrap();+ }+ let report = crate::diagnostic_logs::collect_redacted_from_dirs(&[root.path().into()])+ .await+ .unwrap();+ assert!(report.contains("control_port_bind_failed"));+ assert!(report.contains("port=11435"));+ assert!(report.contains("os error 24"));+ assert!(report.contains("outcome=bind_failed"));+ assert!(!report.contains("unhealthy process"));+ }+#[test]fn startup_only_reclaims_unhealthy_port_owners() {assert!(!should_reclaim_owner(true));diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server.rs b/apps/screenpipe-app-tauri/src-tauri/src/server.rsindex d254e7667..63cce4503 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server.rs@@ -399,46 +399,29 @@ pub async fn run_server(app_handle: tauri::AppHandle, port: u16) {let addr = SocketAddr::from(([127, 0, 0, 1], port));- // main.rs marks a responsive Screenpipe control endpoint before this task- // starts. Preserve that healthy instance; reclaim only an unrelated or- // unresponsive owner left on the control port.- crate::port_conflict::reclaim_owner(+ match crate::port_conflict::bind_control_port(port,crate::port_conflict::healthy_control_server_present(),+ crate::config::app_data_dir(),+ || axum::Server::try_bind(&addr),)- .await;-- // Retry binding with backoff — avoids panic when a previous instance hasn't- // released the port yet (e.g. fast restart, TIME_WAIT on Linux).- let mut last_err = None;- for attempt in 0..5u64 {- if attempt > 0 {- tracing::warn!("port {} in use, retry {}/5...", port, attempt);- tokio::time::sleep(std::time::Duration::from_secs(attempt)).await;- }- match axum::Server::try_bind(&addr) {- Ok(builder) => {- info!("Server listening on {}", addr);- if let Err(e) = builder.serve(app.into_make_service()).await {- error!("server error on {}: {}", addr, e);- }- return;+ .await+ {+ Ok(builder) => {+ info!("Server listening on {}", addr);+ if let Err(e) = builder.serve(app.into_make_service()).await {+ error!("server error on {}: {}", addr, e);}- Err(e) => {- last_err = Some(e);+ }+ Err(error) => {+ error!("failed to bind to {addr}: {error:#}");+ if crate::port_conflict::healthy_control_server_present() {+ crate::port_conflict::show_healthy_screenpipe(&app_handle, port);+ } else {+ crate::port_conflict::show_control_start_failed(&app_handle, port, &error);}}}- error!(- "failed to bind to {} after 5 attempts: {}",- addr,- last_err.map(|e| e.to_string()).unwrap_or_default()- );- if crate::port_conflict::healthy_control_server_present() {- crate::port_conflict::show_healthy_screenpipe(&app_handle, port);- } else {- crate::port_conflict::show_reclaim_failed(&app_handle, port);- }}async fn send_inbox_message(