YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(pi): retry hosted AI capacity contention instead of terminating t… (#5761)
Patel Dhrit8a9cad33cfc7
diff --git a/crates/screenpipe-core/Cargo.toml b/crates/screenpipe-core/Cargo.tomlindex 384c04163..cb7f72326 100644--- a/crates/screenpipe-core/Cargo.toml+++ b/crates/screenpipe-core/Cargo.toml@@ -115,3 +115,6 @@ sha2 = { workspace = true }# Used by sync::client wire-compat test (asserts the PUT request shape# stays byte-identical after the upload_to_s3 → HttpPutDirect migration).wiremock = { workspace = true }+# test-util (virtual clock) lets agents::pi's rate-limit retry loop tests+# drive real minutes of retry/backoff timing without real wall-clock waits.+tokio = { version = "1.15", features = ["test-util"] }diff --git a/crates/screenpipe-core/src/agents/pi.rs b/crates/screenpipe-core/src/agents/pi.rsindex 99eeccefa..feb6da8fb 100644--- a/crates/screenpipe-core/src/agents/pi.rs+++ b/crates/screenpipe-core/src/agents/pi.rs@@ -154,21 +154,43 @@ const MAX_RATE_LIMIT_RETRIES: usize = 3;const RATE_LIMIT_DEFAULT_WAIT_SECS: u64 = 10;/// Cap so an oversized `reset_in` can't stall a pipe run indefinitely.const RATE_LIMIT_MAX_WAIT_SECS: u64 = 60;+/// Total time budget for retrying `hosted_ai_capacity_reserved` contention.+/// Unlike a provider rate limit this error means a sibling chat/pipe is briefly+/// holding the shared hosted-AI slot, not that anything is wrong, so it gets a+/// time budget instead of `MAX_RATE_LIMIT_RETRIES`.+///+/// This must stay well below the run's enclosing timeout+/// (`pipes::DEFAULT_TIMEOUT_SECS`, 600s). Waiting for the gateway's full+/// reservation TTL would let contention alone consume the entire budget, so the+/// outer `tokio::time::timeout` would kill the run at the same moment the slot+/// freed up: the user waits the full timeout and still gets nothing done. A+/// fraction of the timeout keeps contention survivable while leaving the+/// majority of the run for actual work.+const CAPACITY_RESERVED_MAX_WAIT_SECS: u64 = 180;++/// Parse a `"field": <digits>` JSON number out of a pi error payload.+fn parse_json_number_field(text: &str, field: &str) -> Option<u64> {+ let needle = format!("\"{field}\"");+ let idx = text.find(&needle)?;+ let rest = &text[idx + needle.len()..];+ let rest = rest.trim_start_matches(|c: char| c == ':' || c.is_whitespace());+ let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();+ digits.parse::<u64>().ok()+}/// Parse the rate-limit retry hint (in seconds) from a pi error payload.///-/// The cloud gateway returns a 429 body containing `"reset_in":<secs>` plus a-/// human-readable "Please wait N seconds". We prefer the structured `reset_in`-/// field and fall back to the prose. Returns `None` when no hint is present.+/// The cloud gateway returns a 429 body containing `"reset_in":<secs>` for+/// provider rate limits, or `"retry_after_seconds":<secs>` for+/// `hosted_ai_capacity_reserved` contention, plus a human-readable "Please+/// wait N seconds". We prefer the structured fields and fall back to the+/// prose. Returns `None` when no hint is present.fn parse_rate_limit_reset_secs(text: &str) -> Option<u64> {- // Prefer the structured "reset_in" field.- if let Some(idx) = text.find("\"reset_in\"") {- let rest = &text[idx + "\"reset_in\"".len()..];- let rest = rest.trim_start_matches(|c: char| c == ':' || c.is_whitespace());- let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();- if let Ok(secs) = digits.parse::<u64>() {- return Some(secs);- }+ if let Some(secs) = parse_json_number_field(text, "reset_in") {+ return Some(secs);+ }+ if let Some(secs) = parse_json_number_field(text, "retry_after_seconds") {+ return Some(secs);}// Fall back to the human-readable "wait N seconds".let lower = text.to_lowercase();@@ -199,6 +221,129 @@ fn is_rate_limit_error(text: &str) -> bool {|| lower.contains("requests per minute")|| lower.contains("too many requests")|| lower.contains("\"reset_in\"")+ || lower.contains("hosted_ai_capacity_reserved")+}++/// Whether a pi failure is hosted-AI capacity contention+/// (`hosted_ai_capacity_reserved`) rather than a real provider rate limit.+///+/// The shared hosted-AI slot is briefly held by a sibling chat or pipe; the+/// gateway's `retry_after_seconds` hint is honored until it clears or the+/// reservation TTL (`CAPACITY_RESERVED_MAX_WAIT_SECS`) expires, instead of+/// the fixed `MAX_RATE_LIMIT_RETRIES` used for other rate limits.+fn is_capacity_reserved_error(text: &str) -> bool {+ text.to_lowercase().contains("hosted_ai_capacity_reserved")+}++/// Outcome of [`next_rate_limit_retry`]: retry after `wait_secs`, tagged with+/// whether this was hosted-AI capacity contention (for logging/telemetry).+struct RateLimitRetryDecision {+ wait_secs: u64,+ capacity_reserved: bool,+}++/// Decide whether a failed pi run should be retried, and for how long to wait.+///+/// Pure retry policy, isolated from the I/O loop in `run_streaming` so it can+/// be exercised without spawning a real pi subprocess. `plain_rate_limit_retries`+/// and `capacity_reserved_waited_secs` are independent budget counters+/// accumulated *before* this attempt — kept separate so a run that alternates+/// between the two error classes doesn't let one exhaust the other's budget.+/// Returns `None` once the applicable budget is exhausted:+/// `MAX_RATE_LIMIT_RETRIES` attempts for a generic rate limit, or+/// `CAPACITY_RESERVED_MAX_WAIT_SECS` of total wait for+/// `hosted_ai_capacity_reserved` contention.+fn next_rate_limit_retry(+ stderr: &str,+ plain_rate_limit_retries: usize,+ capacity_reserved_waited_secs: u64,+) -> Option<RateLimitRetryDecision> {+ if !is_rate_limit_error(stderr) {+ return None;+ }+ let capacity_reserved = is_capacity_reserved_error(stderr);+ if capacity_reserved {+ if capacity_reserved_waited_secs >= CAPACITY_RESERVED_MAX_WAIT_SECS {+ return None;+ }+ } else if plain_rate_limit_retries >= MAX_RATE_LIMIT_RETRIES {+ return None;+ }+ let wait_secs = parse_rate_limit_reset_secs(stderr)+ .unwrap_or(RATE_LIMIT_DEFAULT_WAIT_SECS)+ .clamp(1, RATE_LIMIT_MAX_WAIT_SECS);+ Some(RateLimitRetryDecision {+ wait_secs,+ capacity_reserved,+ })+}++/// Drive a failed pi run through rate-limit / capacity-contention retries+/// until it succeeds or `next_rate_limit_retry` exhausts the applicable+/// budget.+///+/// Extracted out of `run_streaming` so the retry *loop* — not just the pure+/// `next_rate_limit_retry` policy — can be driven directly in tests via a+/// fake `respawn`, under tokio's virtual clock, instead of only exercising+/// the policy function in isolation.+async fn retry_rate_limited_output<F, Fut>(+ mut output: AgentOutput,+ line_tx: &tokio::sync::mpsc::UnboundedSender<String>,+ mut respawn: F,+) -> Result<AgentOutput>+where+ F: FnMut() -> Fut,+ Fut: std::future::Future<Output = Result<AgentOutput>>,+{+ // `attempt` is a monotonic counter for logging only. The two retry+ // budgets below stay independent so a run that alternates between plain+ // rate limits and capacity contention can't have one exhaust the other's+ // allowance.+ let mut attempt = 0usize;+ let mut plain_rate_limit_retries = 0usize;+ let mut capacity_reserved_waited_secs = 0u64;+ loop {+ if output.success {+ break;+ }+ let Some(decision) = next_rate_limit_retry(+ &output.stderr,+ plain_rate_limit_retries,+ capacity_reserved_waited_secs,+ ) else {+ break;+ };+ attempt += 1;+ let wait_secs = decision.wait_secs;+ if decision.capacity_reserved {+ capacity_reserved_waited_secs += wait_secs;+ warn!(+ "pi hit hosted AI capacity contention (attempt {}, {}/{}s waited), waiting {}s before retry (stderr: {})",+ attempt,+ capacity_reserved_waited_secs,+ CAPACITY_RESERVED_MAX_WAIT_SECS,+ wait_secs,+ output.stderr.trim()+ );+ } else {+ plain_rate_limit_retries += 1;+ warn!(+ "pi rate limited (attempt {}/{}), waiting {}s before retry (stderr: {})",+ plain_rate_limit_retries,+ MAX_RATE_LIMIT_RETRIES,+ wait_secs,+ output.stderr.trim()+ );+ }+ // Surface the wait to any UI/log consumer draining line_tx.+ let _ = line_tx.send(format!(+ r#"{{"type":"status","kind":"rate_limit_retry","wait_secs":{},"attempt":{},"max_attempts":{},"capacity_reserved":{}}}"#,+ wait_secs, attempt, MAX_RATE_LIMIT_RETRIES, decision.capacity_reserved+ ));+ tokio::time::sleep(std::time::Duration::from_secs(wait_secs)).await;+ output = respawn().await?;+ }+ Ok(output)}/// Fetch the model catalog from the Cloudflare Worker gateway and convert@@ -2029,54 +2174,41 @@ impl AgentExecutor for PiExecutor {.await?;}- // Retry on provider rate limiting (HTTP 429). The cloud gateway caps+ // Retry on provider rate limiting (HTTP 429) and on hosted-AI capacity+ // contention (`hosted_ai_capacity_reserved`). The cloud gateway caps// requests per minute; concurrent scheduler pressure or a single busy- // run can trip it. pi exits 0 but surfaces the 429 as an assistant- // error, so `output.success` is false with the payload (including- // "reset_in") in stderr. Honor that hint, wait, and re-run instead of- // failing the whole pipe — which previously left automations silently- // doing nothing. (Runs that legitimately exceed the per-minute budget- // also need scheduler pacing, but a wait-and-retry still beats a hard- // stop.)- let mut rate_limit_retries = 0usize;- while !output.success- && rate_limit_retries < MAX_RATE_LIMIT_RETRIES- && is_rate_limit_error(&output.stderr)- {- rate_limit_retries += 1;- let wait_secs = parse_rate_limit_reset_secs(&output.stderr)- .unwrap_or(RATE_LIMIT_DEFAULT_WAIT_SECS)- .clamp(1, RATE_LIMIT_MAX_WAIT_SECS);- warn!(- "pi rate limited (attempt {}/{}), waiting {}s before retry (stderr: {})",- rate_limit_retries,- MAX_RATE_LIMIT_RETRIES,- wait_secs,- output.stderr.trim()- );- // Surface the wait to any UI/log consumer draining line_tx.- let _ = line_tx.send(format!(- r#"{{"type":"status","kind":"rate_limit_retry","wait_secs":{},"attempt":{},"max_attempts":{}}}"#,- wait_secs, rate_limit_retries, MAX_RATE_LIMIT_RETRIES- ));- tokio::time::sleep(std::time::Duration::from_secs(wait_secs)).await;- output = self- .spawn_pi_streaming(- &pi_path,- prompt,- &resolved_model,- working_dir,- &resolved_provider,- provider_api_key,- None,- line_tx.clone(),- continue_session,- pipe_system_prompt,- mcp_server_allowlist,- session_owner,- )- .await?;- }+ // run can trip it, and the shared hosted-AI slot can briefly be held+ // by a sibling chat or pipe. pi exits 0 but surfaces the 429 as an+ // assistant error, so `output.success` is false with the payload+ // (including "reset_in" or "retry_after_seconds") in stderr. Honor+ // that hint, wait, and re-run instead of failing the whole run —+ // which previously left automations silently doing nothing.+ //+ // Capacity contention isn't a real error — someone else is just using+ // the shared slot — so it gets a time budget+ // (`CAPACITY_RESERVED_MAX_WAIT_SECS`) instead of+ // `MAX_RATE_LIMIT_RETRIES`: contention outlasting three short waits+ // would otherwise kill the run even though the slot frees up shortly+ // after. That budget is deliberately a fraction of the enclosing run+ // timeout so waiting can never consume the whole run.+ // See `retry_rate_limited_output`.+ output = retry_rate_limited_output(output, &line_tx, || {+ self.spawn_pi_streaming(+ &pi_path,+ prompt,+ &resolved_model,+ working_dir,+ &resolved_provider,+ provider_api_key,+ None,+ line_tx.clone(),+ continue_session,+ pipe_system_prompt,+ mcp_server_allowlist,+ session_owner,+ )+ })+ .await?;Ok(output)}@@ -4462,6 +4594,10 @@ mod tests {// Unrelated error carries no hint.assert_eq!(parse_rate_limit_reset_secs("model not found"), None);++ // hosted_ai_capacity_reserved uses "retry_after_seconds", not "reset_in".+ let capacity_reserved = r#"429 {"error":"hosted_ai_capacity_reserved","message":"Other hosted AI chats are still running. Wait for one to finish, then retry.","retry_after_seconds":5}"#;+ assert_eq!(parse_rate_limit_reset_secs(capacity_reserved), Some(5));}#[test]@@ -4671,6 +4807,9 @@ mod tests {"You've exceeded 25 requests per minute"));assert!(is_rate_limit_error(r#"{"reset_in":12}"#));+ assert!(is_rate_limit_error(+ r#"429 {"error":"hosted_ai_capacity_reserved","retry_after_seconds":5}"#+ ));assert!(!is_rate_limit_error("model not found"));assert!(!is_rate_limit_error("credits_exhausted"));assert!(!is_rate_limit_error(r#"429 "daily_cost_limit_exceeded""#));@@ -4680,6 +4819,248 @@ mod tests {));}+ #[test]+ fn test_is_capacity_reserved_error() {+ assert!(is_capacity_reserved_error(+ r#"429 {"error":"hosted_ai_capacity_reserved","retry_after_seconds":5}"#+ ));+ assert!(!is_capacity_reserved_error("HTTP 429 Too Many Requests"));+ assert!(!is_capacity_reserved_error(+ r#"429 "daily_cost_limit_exceeded""#+ ));+ }++ // These exercise `next_rate_limit_retry`, the pure decision function+ // shared by both `run_streaming` call sites (interactive chat and+ // scheduled pipes both execute through it — see the caller in+ // `PiExecutor::run_streaming`) — so one set of cases here covers the+ // "chat" and "pipe executor" scenarios from #5673 without duplicating+ // the same assertions against two entry points.++ #[test]+ fn test_next_rate_limit_retry_capacity_reserved_outlasts_fixed_retry_cap() {+ // Real gateway payload for contention on the shared hosted-AI slot.+ let stderr = r#"429 {"error":"hosted_ai_capacity_reserved","message":"Other hosted AI chats are still running. Wait for one to finish, then retry.","retry_after_seconds":5}"#;++ // Simulate holding capacity for 35s (7 retries at 5s each) — longer+ // than the fixed MAX_RATE_LIMIT_RETRIES=3 a generic rate limit gets.+ let mut retries = 0usize;+ let mut waited = 0u64;+ for _ in 0..7 {+ let decision = next_rate_limit_retry(stderr, retries, waited)+ .expect("capacity contention must keep retrying past the fixed attempt cap");+ assert!(decision.capacity_reserved);+ assert_eq!(decision.wait_secs, 5);+ retries += 1;+ waited += decision.wait_secs;+ }+ assert!(retries > MAX_RATE_LIMIT_RETRIES);+ assert_eq!(waited, 35);+ }++ #[test]+ fn capacity_retry_budget_leaves_room_under_the_run_timeout() {+ // The retry budget and the enclosing pipe timeout were both 600s, so+ // contention alone could burn the entire run: the outer+ // `tokio::time::timeout` would fire at the exact moment the slot freed+ // up and the pipe would do no work at all. Keep the budget a minority+ // of the timeout so a contended run still has time to finish.+ let run_timeout = crate::pipes::DEFAULT_TIMEOUT_SECS;+ assert!(+ CAPACITY_RESERVED_MAX_WAIT_SECS * 2 < run_timeout,+ "capacity retry budget ({CAPACITY_RESERVED_MAX_WAIT_SECS}s) must stay well under the \+ run timeout ({run_timeout}s), or waiting consumes the whole run"+ );+ }++ #[test]+ fn test_next_rate_limit_retry_capacity_reserved_stops_at_reservation_ttl() {+ let stderr = r#"429 {"error":"hosted_ai_capacity_reserved","retry_after_seconds":5}"#;+ assert!(next_rate_limit_retry(stderr, 100, CAPACITY_RESERVED_MAX_WAIT_SECS).is_none());+ assert!(next_rate_limit_retry(stderr, 100, CAPACITY_RESERVED_MAX_WAIT_SECS - 1).is_some());+ }++ #[test]+ fn test_next_rate_limit_retry_generic_rate_limit_still_capped_at_three() {+ let stderr = "HTTP 429 Too Many Requests";+ for retries in 0..MAX_RATE_LIMIT_RETRIES {+ let decision = next_rate_limit_retry(stderr, retries, 0)+ .expect("generic rate limit should retry up to MAX_RATE_LIMIT_RETRIES");+ assert!(!decision.capacity_reserved);+ }+ assert!(next_rate_limit_retry(stderr, MAX_RATE_LIMIT_RETRIES, 0).is_none());+ }++ #[test]+ fn test_next_rate_limit_retry_budgets_are_independent_across_error_classes() {+ // A run that alternates between capacity contention and a plain rate+ // limit must not let one error class's retries consume the other's+ // budget.+ let capacity_stderr =+ r#"429 {"error":"hosted_ai_capacity_reserved","retry_after_seconds":5}"#;+ let plain_stderr = "HTTP 429 Too Many Requests";++ let mut plain_retries = 0usize;+ let mut capacity_waited = 0u64;++ // Three capacity-reserved retries must not touch the plain budget.+ for _ in 0..3 {+ let decision = next_rate_limit_retry(capacity_stderr, plain_retries, capacity_waited)+ .expect("capacity contention should retry");+ assert!(decision.capacity_reserved);+ capacity_waited += decision.wait_secs;+ }+ assert_eq!(plain_retries, 0);++ // The plain rate-limit budget is still fully available afterwards.+ for _ in 0..MAX_RATE_LIMIT_RETRIES {+ let decision = next_rate_limit_retry(plain_stderr, plain_retries, capacity_waited)+ .expect("plain rate limit budget must be untouched by capacity retries");+ assert!(!decision.capacity_reserved);+ plain_retries += 1;+ }+ assert!(next_rate_limit_retry(plain_stderr, plain_retries, capacity_waited).is_none());+ }++ #[test]+ fn test_next_rate_limit_retry_daily_cap_is_terminal() {+ // daily_cost_limit_exceeded must never retry, regardless of counters.+ let stderr = r#"429 "daily_cost_limit_exceeded""#;+ assert!(next_rate_limit_retry(stderr, 0, 0).is_none());+ }++ // The tests below drive `retry_rate_limited_output` itself — the actual+ // loop `run_streaming` calls in production (both interactive chat and+ // scheduled pipes go through it) — rather than only the extracted+ // `next_rate_limit_retry` policy function above. `start_paused` runs+ // tokio's virtual clock, so every real `tokio::time::sleep` the loop+ // performs (including a full simulated 600s reservation TTL) resolves+ // instantly instead of blocking the test for real wall-clock time.++ fn fake_agent_output(stderr: &str, success: bool) -> AgentOutput {+ AgentOutput {+ stdout: String::new(),+ stderr: stderr.to_string(),+ success,+ pid: None,+ }+ }++ #[tokio::test(start_paused = true)]+ async fn test_retry_rate_limited_output_capacity_reserved_succeeds_after_more_than_three_retries(+ ) {+ // Simulate the shared hosted-AI slot staying busy for 4 attempts —+ // one more than MAX_RATE_LIMIT_RETRIES=3 — before it frees up.+ let capacity_stderr = r#"429 {"error":"hosted_ai_capacity_reserved","message":"Other hosted AI chats are still running. Wait for one to finish, then retry.","retry_after_seconds":5}"#;+ let calls = std::cell::RefCell::new(0u32);+ let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();++ let result =+ retry_rate_limited_output(fake_agent_output(capacity_stderr, false), &tx, || {+ *calls.borrow_mut() += 1;+ let attempt = *calls.borrow();+ async move {+ Ok(if attempt <= 4 {+ fake_agent_output(capacity_stderr, false)+ } else {+ fake_agent_output("", true)+ })+ }+ })+ .await+ .expect("retry loop must not error");++ assert!(result.success);+ assert_eq!(*calls.borrow(), 5);+ assert!(*calls.borrow() as usize > MAX_RATE_LIMIT_RETRIES);++ // The status channel actually received one event per retry — proves+ // the line_tx wiring executed, not just the retry decision.+ drop(tx);+ let mut status_events = 0;+ while let Some(msg) = rx.recv().await {+ assert!(msg.contains("rate_limit_retry"));+ assert!(msg.contains("\"capacity_reserved\":true"));+ status_events += 1;+ }+ assert_eq!(status_events, 5);+ }++ #[tokio::test(start_paused = true)]+ async fn test_retry_rate_limited_output_generic_rate_limit_gives_up_after_max_retries() {+ let plain_stderr = "HTTP 429 Too Many Requests";+ let calls = std::cell::RefCell::new(0u32);+ let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();++ let result = retry_rate_limited_output(fake_agent_output(plain_stderr, false), &tx, || {+ *calls.borrow_mut() += 1;+ async move { Ok(fake_agent_output(plain_stderr, false)) }+ })+ .await+ .expect("retry loop must not error even when it gives up");++ assert!(!result.success);+ assert_eq!(*calls.borrow() as usize, MAX_RATE_LIMIT_RETRIES);+ }++ #[tokio::test(start_paused = true)]+ async fn test_retry_rate_limited_output_daily_cap_never_retries() {+ let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();+ let calls = std::cell::RefCell::new(0u32);++ let result = retry_rate_limited_output(+ fake_agent_output(r#"429 "daily_cost_limit_exceeded""#, false),+ &tx,+ || {+ *calls.borrow_mut() += 1;+ async move { Ok(fake_agent_output("", true)) }+ },+ )+ .await+ .expect("retry loop must not error");++ assert!(!result.success);+ assert_eq!(+ *calls.borrow(),+ 0,+ "must not retry a terminal daily-cap error"+ );+ }++ #[tokio::test(start_paused = true)]+ async fn test_retry_rate_limited_output_capacity_reserved_stops_at_reservation_ttl_live() {+ // The gateway's hint (100s here) is clamped to RATE_LIMIT_MAX_WAIT_SECS+ // (60s) — same safety clamp a generic rate limit gets, so an oversized+ // hint can't stall a run — leaving 60s waits until the budget is spent.+ // This is exactly the kind of interaction between two+ // independently-reasonable pieces of logic that only a live run of the+ // loop (not the isolated policy function) catches: this assertion's+ // first draft assumed 100s waits, and the live test caught the clamp.+ // Derived from the constants so retuning the budget updates the+ // expectation instead of silently failing here.+ let expected_retries = CAPACITY_RESERVED_MAX_WAIT_SECS / RATE_LIMIT_MAX_WAIT_SECS;+ let expected_elapsed = expected_retries * RATE_LIMIT_MAX_WAIT_SECS;+ let capacity_stderr = r#"429 {"error":"hosted_ai_capacity_reserved","message":"still busy","retry_after_seconds":100}"#;+ let calls = std::cell::RefCell::new(0u32);+ let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();++ let start = tokio::time::Instant::now();+ let result =+ retry_rate_limited_output(fake_agent_output(capacity_stderr, false), &tx, || {+ *calls.borrow_mut() += 1;+ async move { Ok(fake_agent_output(capacity_stderr, false)) }+ })+ .await+ .expect("retry loop must not error");++ assert!(!result.success);+ assert_eq!(*calls.borrow(), expected_retries as u32);+ assert_eq!(+ start.elapsed(),+ std::time::Duration::from_secs(expected_elapsed)+ );+ }+#[test]fn custom_provider_compat_repairs_ai_genesis_and_overrides_sdk_user_agent() {for base_url in ["https://ai.ai-genesis.app", "https://api.ai-genesis.app/"] {diff --git a/crates/screenpipe-core/src/pipes/mod.rs b/crates/screenpipe-core/src/pipes/mod.rsindex be7368434..d6c2733ca 100644--- a/crates/screenpipe-core/src/pipes/mod.rs+++ b/crates/screenpipe-core/src/pipes/mod.rs@@ -2218,7 +2218,7 @@ pub type ConnectionCheck = Arc<>;/// Default execution timeout: 10 minutes.-const DEFAULT_TIMEOUT_SECS: u64 = 600;+pub(crate) const DEFAULT_TIMEOUT_SECS: u64 = 600;/// Cooldown held after a scheduled run finishes *when other scheduled pipes/// are still queued behind the execution semaphore*. Spaces consecutive runs