Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(runtime): run a timed-out first-run probe once more before failing (#2380)

Max535b76c4f2d6
diff --git a/core/install_integrity.py b/core/install_integrity.py
index f757d1c70..6ca1b25bd 100644
--- a/core/install_integrity.py
+++ b/core/install_integrity.py
@@ -14,6 +14,7 @@ import csv
import hashlib
import importlib.metadata as importlib_metadata
import json
+import logging
import os
import subprocess
import tempfile
@@ -30,6 +31,11 @@ from packaging.utils import canonicalize_name
RUNTIME_DISTRIBUTIONS = ("avibe-os", "vibe-remote")
RUNTIME_PROBE_PREFIX = "avibe-runtime-distributions:"
+# How many runs of one probe may time out before the timeout is raised. See
+# ``run_isolated_probe``.
+PROBE_ATTEMPTS = 2
+
+logger = logging.getLogger(__name__)
@dataclass(frozen=True)
@@ -71,18 +77,36 @@ def isolated_probe_environment() -> dict[str, str]:
def run_isolated_probe(command: Sequence[str], *, timeout: float) -> subprocess.CompletedProcess[str]:
- """Run a candidate probe from a private empty working directory."""
-
- with tempfile.TemporaryDirectory(prefix="avibe-integrity-") as working_directory:
- return subprocess.run(
- command,
- capture_output=True,
- text=True,
- timeout=timeout,
- check=False,
- cwd=working_directory,
- env=isolated_probe_environment(),
- )
+ """Run a candidate probe from a private empty working directory.
+
+ A probe usually runs a tree that was written moments ago. Its first run pays
+ for first reads and the system's first scan of each new executable, and a
+ timeout alone cannot tell that cold start from a hang. That cost is paid
+ once and survives the kill, while a hang repeats, so a timed-out run is run
+ once more and only a second timeout is raised (#2316). A probe that exits,
+ however it exits, is returned as is. The desktop shell's endpoint discovery
+ follows the same rule (``ENDPOINT_ATTEMPTS`` in
+ ``desktop/runtime-host/src/launcher.rs``).
+ """
+
+ attempt = 1
+ while True:
+ try:
+ with tempfile.TemporaryDirectory(prefix="avibe-integrity-") as working_directory:
+ return subprocess.run(
+ command,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ check=False,
+ cwd=working_directory,
+ env=isolated_probe_environment(),
+ )
+ except subprocess.TimeoutExpired:
+ if attempt >= PROBE_ATTEMPTS:
+ raise
+ attempt += 1
+ logger.warning("probe %s timed out after %ss; running it once more", command[0], timeout)
def site_packages_for_python(python_executable: str | os.PathLike[str]) -> list[Path]:
diff --git a/desktop/runtime-host/src/launcher.rs b/desktop/runtime-host/src/launcher.rs
index e075063d8..bf9a27962 100644
--- a/desktop/runtime-host/src/launcher.rs
+++ b/desktop/runtime-host/src/launcher.rs
@@ -64,7 +64,7 @@ const START_ARGS: [&str; 2] = ["start", "--no-open-browser"];
const HAND_OVER_ARG: &str = "--hand-over";
const ENDPOINT_ARGS: [&str; 3] = ["desktop", "endpoint", "--json"];
const REMOVE_BACKENDS_ARGS: [&str; 2] = ["desktop", "remove-backends"];
-/// How long the shell waits for the Runtime to name its own address.
+/// How long one run of the endpoint helper may take before it is killed.
///
/// Sized for a *cold* first launch, not a warm one. The bundle has just been
/// extracted, so nothing in the runtime tree has been paged in or evaluated by
@@ -78,10 +78,26 @@ const REMOVE_BACKENDS_ARGS: [&str; 2] = ["desktop", "remove-backends"];
/// This is the last step on the cold path that still held a warm number --
/// readiness already allows 120s (`DEFAULT_READY_TIMEOUT`). Staying under that
/// keeps discovery from dominating the launch, while 3x the worst measurement
-/// leaves room for slower hardware. The budget is only spent in full when the
-/// endpoint is genuinely broken, and that failure is already retryable.
+/// leaves room for slower hardware. Slower storage than that is what
+/// `ENDPOINT_ATTEMPTS` is for.
const ENDPOINT_TIMEOUT: Duration = Duration::from_secs(60);
+/// How many runs of the endpoint helper may time out before discovery fails.
+///
+/// No fixed budget fits every disk: an update on an IO-starved machine (#2316)
+/// spent 50.7s on this call in one run and was killed at 60s in another, and
+/// the window then showed a failure page while the previous Runtime kept
+/// serving. A timeout alone cannot tell a cold tree from a hung helper, but a
+/// second run can: what made the first one slow -- first reads, and the
+/// system's first scan of each new executable -- is paid once and survives the
+/// kill, while a hang repeats. So a timed-out run is run once more, and only a
+/// second timeout counts as a failure. A helper that exits or answers badly
+/// failed for a reason a repeat would not change, so it is never repeated.
+///
+/// `run_isolated_probe` in `core/install_integrity.py` applies the same rule
+/// to the CLI's probes of a freshly installed tree.
+const ENDPOINT_ATTEMPTS: u32 = 2;
+
/// How much of a helper's stderr the shell keeps: the bootstrap log's
/// diagnostic, and a lifecycle verb's closing JSON verdict.
const MAX_STDERR_TAIL_BYTES: usize = 4 * 1024;
@@ -679,6 +695,8 @@ struct EndpointDescriptor {
/// carry: how long it took, what it printed on stderr, and the exit status it
/// actually had rather than the fact that it was not zero.
struct EndpointAttempt {
+ /// 1 for the first run of this discovery, 2 for its one repeat.
+ number: u32,
result: Result<LoopbackOrigin, LaunchError>,
status: Option<std::process::ExitStatus>,
stderr: Option<Vec<u8>>,
@@ -696,6 +714,21 @@ fn query_endpoint_within(
runtime: &RuntimeCommand,
log: &BootstrapLog,
timeout: Duration,
+) -> Result<LoopbackOrigin, LaunchError> {
+ let mut attempt = 1;
+ loop {
+ match query_endpoint_once(runtime, log, timeout, attempt) {
+ Err(LaunchError::EndpointTimeout) if attempt < ENDPOINT_ATTEMPTS => attempt += 1,
+ result => return result,
+ }
+ }
+}
+
+fn query_endpoint_once(
+ runtime: &RuntimeCommand,
+ log: &BootstrapLog,
+ timeout: Duration,
+ attempt: u32,
) -> Result<LoopbackOrigin, LaunchError> {
let started = Instant::now();
let mut status = None;
@@ -704,14 +737,15 @@ fn query_endpoint_within(
// Collected after the query returns, so a timeout that had to kill the child
// still reports what that child said before it was killed.
let stderr = stderr_reader.and_then(|reader| reader.recv_timeout(STDERR_DRAIN).ok());
- let attempt = EndpointAttempt {
+ let run = EndpointAttempt {
+ number: attempt,
result,
status,
stderr,
elapsed: started.elapsed(),
};
- record_endpoint_attempt(log, &attempt);
- attempt.result
+ record_endpoint_attempt(log, &run);
+ run.result
}
fn endpoint_descriptor(
@@ -812,6 +846,7 @@ fn record_endpoint_attempt(log: &BootstrapLog, attempt: &EndpointAttempt) {
let mut fields = vec![
("outcome", endpoint_outcome(&attempt.result).to_owned()),
("ms", attempt.elapsed.as_millis().to_string()),
+ ("attempt", attempt.number.to_string()),
];
if let Some(status) = attempt.status {
fields.push((
@@ -2050,7 +2085,16 @@ mod tests {
assert!(matches!(failure, Err(LaunchError::EndpointTimeout)));
let written = std::fs::read_to_string(dir.join("bootstrap.log")).expect("the attempt is recorded");
- assert!(written.contains("outcome=\"timeout\""), "{written}");
+ let records: Vec<&str> = written.lines().collect();
+ assert_eq!(
+ records.len(),
+ 2,
+ "a hang is repeated once, then reported, never retried without end: {written}"
+ );
+ for (record, attempt) in records.iter().zip(["1", "2"]) {
+ assert!(record.contains("outcome=\"timeout\""), "{written}");
+ assert!(record.contains(&format!("attempt=\"{attempt}\"")), "{written}");
+ }
assert!(
!written.contains(" exit="),
"a child that was killed has no exit status to report: {written}"
@@ -2063,6 +2107,62 @@ mod tests {
std::fs::remove_dir_all(&dir).ok();
}
+ /// #2316: on an IO-starved disk the first run of a freshly extracted tree
+ /// outlived the whole budget, and the update stopped on a failure page. What
+ /// made that run slow is paid once, so the run after the kill answers.
+ ///
+ /// The fixture's first query stands in for the cold run: it leaves the flag
+ /// behind and then blocks past the deadline. Every later query finds the flag
+ /// and answers. The warm-up run (no arguments) pays this executable's own
+ /// first-exec cost outside the deadline, for the reason given in the test
+ /// above.
+ #[cfg(unix)]
+ #[test]
+ fn a_cold_endpoint_killed_at_its_deadline_answers_on_the_repeat() {
+ let dir = scratch_dir("endpoint-cold-repeat");
+ let paid = dir.join("paid");
+ let executable = write_fake_runtime(
+ &dir,
+ &format!(
+ "#!/bin/sh\n[ \"$1\" = desktop ] || exit 0\nif [ -e {flag} ]; then printf '%s\\n' '{{\"schema_version\":1,\"origin\":\"http://127.0.0.1:6123\"}}'; exit 0; fi\n: > {flag}\nexec sleep 30\n",
+ flag = paid.display()
+ ),
+ );
+ let warm_up = Command::new(&executable)
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()
+ .expect("the fixture runs");
+ assert!(
+ warm_up.success() && !paid.exists(),
+ "the warm-up run must not pay the cold run"
+ );
+ let log = BootstrapLog::at(dir.join("bootstrap.log"));
+
+ let endpoint = query_endpoint_within(
+ &RuntimeCommand::installed(executable),
+ &log,
+ Duration::from_millis(1_500),
+ )
+ .expect("the run after a timed-out cold run answers");
+
+ assert_eq!(endpoint.as_str(), "http://127.0.0.1:6123");
+ let written = std::fs::read_to_string(dir.join("bootstrap.log")).expect("both runs are recorded");
+ let records: Vec<&str> = written.lines().collect();
+ assert_eq!(records.len(), 2, "{written}");
+ assert!(
+ records[0].contains("outcome=\"timeout\"") && records[0].contains("attempt=\"1\""),
+ "{written}"
+ );
+ assert!(
+ records[1].contains("outcome=\"ok\"") && records[1].contains("attempt=\"2\""),
+ "{written}"
+ );
+
+ std::fs::remove_dir_all(&dir).ok();
+ }
+
/// Reused against extracted is the line that settles whether an install is
/// quietly reinstalling itself on every launch.
#[test]
diff --git a/tests/test_install_integrity.py b/tests/test_install_integrity.py
index 7654c3fd4..f07b65722 100644
--- a/tests/test_install_integrity.py
+++ b/tests/test_install_integrity.py
@@ -3,9 +3,12 @@ import hashlib
import json
import stat
import subprocess
+import sys
from pathlib import Path
from types import SimpleNamespace
+import pytest
+
from config.platform_registry import platform_descriptors
from core import install_integrity
from core.install_integrity import verify_site_packages
@@ -92,6 +95,41 @@ def test_verify_python_environment_probes_use_private_empty_directories(monkeypa
assert calls[0]["cwd"] != calls[1]["cwd"]
+# Each run appends one line to the runs file. A "cold" probe stands in for
+# the first run of a freshly written tree: it blocks past the deadline once,
+# and every later run answers. A "hung" probe never answers.
+_PROBE_SCRIPT = """
+import pathlib, sys, time
+runs = pathlib.Path(sys.argv[1])
+with runs.open("a") as handle:
+ handle.write("run\\n")
+if sys.argv[2] == "cold" and len(runs.read_text().splitlines()) > 1:
+ print("answered")
+ sys.exit(0)
+time.sleep(60)
+"""
+
+
+ ("behaviour", "answers", "runs"),
+ [("cold", True, 2), ("hung", False, 2)],
+)
+def test_a_timed_out_probe_runs_once_more_before_the_timeout_is_final(tmp_path, behaviour, answers, runs):
+ """#2316: a cold first run outlived its fixed budget and failed the install."""
+
+ runs_file = tmp_path / "runs"
+ command = [sys.executable, "-c", _PROBE_SCRIPT, str(runs_file), behaviour]
+
+ if answers:
+ result = install_integrity.run_isolated_probe(command, timeout=3)
+ assert result.returncode == 0
+ assert result.stdout.strip() == "answered"
+ else:
+ with pytest.raises(subprocess.TimeoutExpired):
+ install_integrity.run_isolated_probe(command, timeout=3)
+ assert len(runs_file.read_text().splitlines()) == runs
+
+
def test_candidate_probe_does_not_inherit_pythonpath(monkeypatch, tmp_path):
site_packages = tmp_path / "lib" / "python3.12" / "site-packages"
_write_record(site_packages, "demo_pkg/__init__.py", b"value = 1\n")