Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(tray): 等待慢启动服务并补齐迟到就绪记录

longmeidao7e72503fba4d
diff --git a/scripts/restart_windows_tray.py b/scripts/restart_windows_tray.py
index 8a9fedfe3..c0ca608a6 100644
--- a/scripts/restart_windows_tray.py
+++ b/scripts/restart_windows_tray.py
@@ -81,7 +81,12 @@ def main(argv: list[str] | None = None) -> int:
failed = ";".join(step["message"] for step in startup.get("steps", ())
if step.get("state") in ("failed", "stale"))
payload["ok"] = False
- payload["message"] += f";新托盘启动准备失败:{failed}"
+ if failed:
+ payload["message"] += f";新托盘启动准备失败:{failed}"
+ elif any(step.get("state") == "pending" for step in startup.get("steps", ())):
+ payload["message"] += ";新托盘服务仍在启动,请查看托盘状态"
+ else:
+ payload["message"] += ";新托盘启动准备未完成"
print(json.dumps(payload, ensure_ascii=False))
return 0 if payload["ok"] else 1
diff --git a/src/peach/runtime_prepare.py b/src/peach/runtime_prepare.py
index 15b6b87b4..f96310ef6 100644
--- a/src/peach/runtime_prepare.py
+++ b/src/peach/runtime_prepare.py
@@ -54,7 +54,7 @@ Run = Callable[..., subprocess.CompletedProcess]
@dataclass(frozen=True)
class Step:
- """一步准备的结果。`state` 取 skipped / current / stale / synced / applied / failed。"""
+ """一步准备的结果。`state` 取 skipped / pending / current / stale / synced / applied / failed。"""
name: str
state: str
@@ -257,7 +257,7 @@ def record(state_dir: Path, steps: Iterable[Step], *, pid: int | None = None) ->
payload = {
"pid": os.getpid() if pid is None else pid,
"at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
- "ok": not any(step.problem for step in items),
+ "ok": not any(step.problem or step.state == "pending" for step in items),
"steps": [asdict(step) for step in items],
}
directory = Path(state_dir)
diff --git a/src/peach/tray.py b/src/peach/tray.py
index 8e87af0af..b2e59a990 100644
--- a/src/peach/tray.py
+++ b/src/peach/tray.py
@@ -280,6 +280,7 @@ class ServiceManager:
self._ledger_plan = ledger_plan or self._current_ledger_plan
self._mount_share = mount_share or self._mount_shared_root
self._owned: dict[str, subprocess.Popen] = {}
+ self._starting: set[str] = set()
# 子服务都挂进这个 Job:托盘被强杀时句柄由内核关闭,整棵 `peach serve` 一起收掉,
# 不会留下孤儿继续占着 80/443,让下一份托盘见端口健康就不拉自己的。
self._job = None
@@ -299,6 +300,9 @@ class ServiceManager:
response = self._health_get(
spec.health_url, timeout=0.5, verify=spec.verify, trust_env=False,
)
+ # 已经能应答的服务进入正常健康判定;503 或坏响应不能一直算「启动中」。
+ with self._lock:
+ self._starting.discard(spec.name)
payload = response.json()
ok = response.status_code == 200 and payload.get("ok") is True
detail = "" if ok else f"状态码 {response.status_code}"
@@ -318,6 +322,17 @@ class ServiceManager:
def log_dir(self) -> Path:
return self._log_dir
+ def starting_services(self) -> tuple[str, ...]:
+ """尚未应答、且本托盘刚拉起的进程仍活着的服务。"""
+ with self._lock:
+ return tuple(name for name in self._starting
+ if name in self._owned and self._owned[name].poll() is None)
+
+ def waiting_for_startup(self) -> bool:
+ with self._lock:
+ down = {spec.name for spec in self.specs if not self._last_health[spec.name][0]}
+ return bool(down) and down <= set(self.starting_services())
+
def status(self) -> str:
"""菜单里那一行状态:每个服务逐个点名,正常的和异常的都写出来。
@@ -329,6 +344,8 @@ class ServiceManager:
for spec in self.specs:
ok, detail = self._last_health[spec.name]
state = (f"正常({detail})" if detail else "正常") if ok else f"异常({detail})" if detail else "异常"
+ if not ok and spec.name in self.starting_services():
+ state = "启动中"
parts.append(f"{spec.name.upper()} {state}")
return " · ".join(parts)
@@ -339,6 +356,8 @@ class ServiceManager:
warnings = set(value for spec in self.specs
for value in self._mount_warnings.get(spec.name, ()))
state = ("运行中" if healthy == len(self.specs) else "部分服务异常") if healthy else "服务未运行"
+ if self.waiting_for_startup():
+ state = "启动中"
return f"{state} · {len(warnings)} 项提醒" if warnings else state
def status_details(self) -> tuple[str, ...]:
@@ -348,6 +367,8 @@ class ServiceManager:
for spec in self.specs:
ok, detail = self._last_health[spec.name]
state = "正常" if ok else f"异常({detail})" if detail else "异常"
+ if not ok and spec.name in self.starting_services():
+ state = "启动中"
services.append(menu_text(f"{spec.name.upper()} {state}"))
warnings = dict.fromkeys(value for spec in self.specs
for value in self._mount_warnings.get(spec.name, ()))
@@ -411,6 +432,7 @@ class ServiceManager:
)
assign_to_job(self._job, process)
self._owned[spec.name] = process
+ self._starting.add(spec.name)
logging.getLogger(__name__).info(
"拉起 %s 服务,PID %s", spec.name, getattr(process, "pid", "?"))
@@ -434,6 +456,7 @@ class ServiceManager:
processes = [process] if process is not None else []
stopped = [name]
for key in stopped:
+ self._starting.discard(key)
if key in self._last_health:
self._last_health[key] = (False, "已停止")
for process in processes:
@@ -1071,6 +1094,7 @@ class PeachTray:
self._diagnostics_lock = threading.Lock()
self._build_checked_at = time.monotonic()
self._build_attempted_head: str | None = None
+ self._pending_startup: tuple[list, bool, str | None] | None = None
version_menu = pystray.Menu(
pystray.MenuItem(lambda _item: f"Peach {self.version.package_version}", None, enabled=False),
pystray.MenuItem(lambda _item: self.version.build_label, None, enabled=False),
@@ -1152,6 +1176,27 @@ class PeachTray:
if step.state == "stale" else "")
for step in steps if step.problem]
+ def _complete_startup(self, ready: bool, steps: list, *, open_browser: bool = False,
+ source_commit: str | None = None) -> list[str]:
+ """记录当前结果;观察窗口结束后仍在启动的服务由健康监测补齐。"""
+ if ready:
+ readiness = self._readiness() or runtime_prepare.Step("readiness", "current", "服务已就绪")
+ elif self.manager.waiting_for_startup():
+ readiness = runtime_prepare.Step("readiness", "pending", "服务仍在启动,请查看托盘状态")
+ else:
+ readiness = runtime_prepare.Step("readiness", "failed", "服务未能启动,请查看托盘状态和日志")
+ self._pending_startup = ((steps, open_browser, source_commit)
+ if readiness.state == "pending" else None)
+ problems = self._record([*steps, readiness])
+ if readiness.state == "current":
+ if source_commit is not None:
+ pending = self.windows_updates.pending()
+ if pending is not None and pending.commit == source_commit:
+ self.windows_updates.clear_pending()
+ if open_browser:
+ self.gate.open()
+ return problems
+
def _restart_whole_tray(self, tray_icon, reason) -> bool:
"""依赖要换就整托盘重启:本进程加载着 venv 里的 `.pyd`,活着的托盘里同步不了。
@@ -1177,7 +1222,9 @@ class PeachTray:
tray_icon.notify(f"整体重启托盘没有完成:{message}。只重启子服务。", "Peach")
return False
- def _restart_services(self, tray_icon) -> tuple[bool | None, list[str]]:
+ def _restart_services(
+ self, tray_icon, *, source_commit: str | None = None,
+ ) -> tuple[bool | None, list[str]]:
"""停子服务、迁移、再拉起并检查就绪。返回 (是否恢复, 要告诉人的问题)。
依赖与锁文件不一致时先交给整托盘重启,交出去了就返回 (None, [])。
@@ -1192,12 +1239,12 @@ class PeachTray:
migrated.append(step)
if step is not None and step.failed:
raise RuntimeError(step.message)
+ self._pending_startup = None
try:
ready = self.manager.restart(prepare=prepare if self.preparation else None)
except RuntimeError:
return False, self._record([dependencies, *migrated])
- readiness = [self._readiness()] if ready else []
- return ready, self._record([dependencies, *migrated, *readiness])
+ return ready, self._complete_startup(ready, [dependencies, *migrated], source_commit=source_commit)
def restart(self, icon=None, _item=None) -> None:
if not self._action_lock.acquire(blocking=False):
@@ -1213,7 +1260,9 @@ class PeachTray:
self.manager.stop_owned()
return
tray_icon.update_menu()
- if not ready:
+ if self._pending_startup is not None:
+ tray_icon.notify("服务仍在启动,请查看托盘状态。", "Peach")
+ elif not ready:
tray_icon.notify(";".join(["服务未能在 20 秒内恢复,请查看日志", *problems])
+ "。", "Peach")
elif problems:
@@ -1356,12 +1405,13 @@ class PeachTray:
tray_icon.stop()
return
- ready, problems = self._restart_services(tray_icon)
+ ready, problems = self._restart_services(tray_icon, source_commit=pending.commit)
if ready is None:
return
- if ready:
- self.windows_updates.clear_pending()
- suffix = "服务已重启。" if ready else "服务未能恢复,请查看日志。"
+ if self._pending_startup is not None:
+ suffix = "服务仍在启动,请查看托盘状态。"
+ else:
+ suffix = "服务已重启。" if ready else "服务未能恢复,请查看日志。"
if problems:
suffix += "但" + ";".join(problems) + "。"
tray_icon.update_menu()
@@ -1420,6 +1470,17 @@ class PeachTray:
if self.gate.poll() and self.preparation is not None:
self.preparation = tray_preparation(self.gate.config, self.manager)
down = [spec for spec in self.manager.specs if not self.manager.healthy(spec)]
+ if self._pending_startup is not None and self._action_lock.acquire(blocking=False):
+ try:
+ if self._pending_startup is not None:
+ steps, open_browser, source_commit = self._pending_startup
+ problems = self._complete_startup(
+ not down, steps, open_browser=open_browser, source_commit=source_commit,
+ )
+ if problems and self._pending_startup is None:
+ self.icon.notify(";".join(problems), "Peach")
+ finally:
+ self._action_lock.release()
if down:
self.revive_services()
self.icon.update_menu()
@@ -1468,19 +1529,10 @@ class PeachTray:
except Exception:
logging.getLogger(__name__).warning("数据库备份清退失败", exc_info=True)
self._startup_warning = None
- if self.gate.waiting:
- if self.manager.wait_until_ready():
- if not self.silent:
- self.gate.open()
- else:
- self._startup_warning = "首次设置服务没能启动,请查看日志。"
- elif not self.manager.wait_until_ready():
- self._startup_warning = "Peach 只启动了部分服务,请查看托盘状态和日志。"
- else:
- startup.append(self._readiness())
- if self.show_browser:
- self.gate.open()
- problems = self._record(startup)
+ problems = self._complete_startup(
+ self.manager.wait_until_ready(), startup,
+ open_browser=(not self.silent if self.gate.waiting else self.show_browser),
+ )
if problems:
self._startup_warning = ";".join(
part for part in (self._startup_warning, *problems) if part)
diff --git a/tests/test_restart_windows_tray.py b/tests/test_restart_windows_tray.py
index b747d7948..395fa91fa 100644
--- a/tests/test_restart_windows_tray.py
+++ b/tests/test_restart_windows_tray.py
@@ -480,6 +480,16 @@ class RestartEntryTests(unittest.TestCase):
self.assertEqual(code, 0)
self.assertIn("未取得新托盘的启动准备记录", printed["message"])
+ def test_pending_readiness_is_reported_as_starting_not_failed(self):
+ self.startup = {"pid": 30, "ok": False, "steps": [
+ {"name": "readiness", "state": "pending", "message": "服务仍在启动"}]}
+ code, printed, _restart = self.run_entry(
+ [], windows_restart.RestartResult(True, "源码托盘已重启", new_tray_pid=30))
+ self.assertEqual(code, 1)
+ self.assertFalse(printed["ok"])
+ self.assertIn("仍在启动", printed["message"])
+ self.assertNotIn("准备失败", printed["message"])
+
def test_the_entry_hands_the_package_to_swap_and_prints_the_outcome(self):
entry = load_entry()
finished = windows_restart.RestartResult(True, "重启完成", backup="C:/dist/backup.exe")
diff --git a/tests/test_runtime_prepare.py b/tests/test_runtime_prepare.py
index 1736bbb42..8200f49ce 100644
--- a/tests/test_runtime_prepare.py
+++ b/tests/test_runtime_prepare.py
@@ -262,6 +262,13 @@ class RecordTests(unittest.TestCase):
written = record(Path(directory), [Step("dependencies", "stale", "不一致")], pid=1)
self.assertFalse(written["ok"])
+ def test_pending_readiness_is_incomplete_without_being_a_failure(self):
+ with tempfile.TemporaryDirectory() as directory:
+ step = Step("readiness", "pending", "服务仍在启动")
+ written = record(Path(directory), [step], pid=1)
+ self.assertFalse(written["ok"])
+ self.assertFalse(step.problem)
+
class TaskGateTests(unittest.TestCase):
HOST = "peach-win"
diff --git a/tests/test_tray.py b/tests/test_tray.py
index 19326d878..f65f6d9cf 100644
--- a/tests/test_tray.py
+++ b/tests/test_tray.py
@@ -1388,6 +1388,45 @@ class ServiceRevivalTests(unittest.TestCase):
self.assertEqual(len(handed), 2)
self.assertTrue(all(handle.closed for handle in handed))
+ def test_an_owned_live_child_starts_until_its_first_response_then_uses_health(self):
+ process = Mock()
+ process.poll.return_value = None
+ get = Mock(side_effect=OSError("not listening"))
+ with tempfile.TemporaryDirectory() as directory:
+ manager = ServiceManager((self.SPEC,), popen=Mock(return_value=process),
+ log_dir=Path(directory), health_get=get)
+ with patch("peach.tray.assign_to_job"), patch("peach.tray.create_kill_on_close_job"):
+ manager.start_missing()
+ manager.start_missing()
+ manager._popen.assert_called_once()
+ self.assertTrue(manager.waiting_for_startup())
+ self.assertEqual(manager.status_summary(), "启动中")
+ self.assertIn("HTTPS 启动中", manager.status_details())
+ get.side_effect = None
+ get.return_value = ReadyResponse(200, {"ok": True})
+ self.assertTrue(manager.healthy(self.SPEC))
+ get.side_effect = OSError("lost listener")
+ self.assertFalse(manager.healthy(self.SPEC))
+ self.assertFalse(manager.waiting_for_startup())
+ self.assertIn("异常", manager.status())
+
+ def test_an_exited_child_or_an_error_response_is_not_starting(self):
+ process = Mock()
+ process.poll.return_value = None
+ get = Mock(side_effect=OSError("not listening"))
+ with tempfile.TemporaryDirectory() as directory:
+ manager = ServiceManager((self.SPEC,), popen=Mock(return_value=process),
+ log_dir=Path(directory), health_get=get)
+ with patch("peach.tray.assign_to_job"), patch("peach.tray.create_kill_on_close_job"):
+ manager.start_missing()
+ process.poll.return_value = 1
+ self.assertFalse(manager.waiting_for_startup())
+ process.poll.return_value = None
+ get.side_effect = None
+ get.return_value = ReadyResponse(503, {"ok": False})
+ self.assertFalse(manager.healthy(self.SPEC))
+ self.assertFalse(manager.waiting_for_startup())
+
class TrayLogFileTests(unittest.TestCase):
def test_the_file_keeps_tray_events_and_drops_per_probe_http_noise(self):
@@ -1471,6 +1510,7 @@ class TrayPreparationTests(unittest.TestCase):
manager.specs = ()
manager.start_missing.side_effect = lambda: self.events.append("start")
manager.wait_until_ready.return_value = ready
+ manager.waiting_for_startup.return_value = False
manager.readiness_problems.return_value = list(problems)
def restart(prepare=None):
@@ -1527,6 +1567,100 @@ class TrayPreparationTests(unittest.TestCase):
self.assertIn("数据库结构与迁移不一致", tray._startup_warning)
self.assertFalse(runtime_prepare.read_record(self.root / "state", pid=os.getpid())["ok"])
+ def test_late_readiness_completes_the_record_and_opens_the_browser_once(self):
+ manager = self.manager(ready=False)
+ manager.specs = (ServiceRevivalTests.SPEC,)
+ manager.waiting_for_startup.return_value = True
+ manager.healthy.return_value = True
+ tray = self.tray(manager, self.preparation())
+ tray.show_browser = True
+ self.run_tray(tray)
+ record = runtime_prepare.read_record(self.root / "state", pid=os.getpid())
+ self.assertFalse(record["ok"])
+ self.assertEqual(record["steps"][-1]["state"], "pending")
+ self.assertIsNone(tray._startup_warning)
+ tray.gate.open.assert_not_called()
+ tray.gate.poll.return_value = False
+ for _ in range(2):
+ ServiceRevivalTests.one_health_round(self, tray)
+ self.assertTrue(runtime_prepare.read_record(self.root / "state", pid=os.getpid())["ok"])
+ tray.gate.open.assert_called_once_with()
+ manager.start_missing.assert_called_once_with()
+
+ def test_first_setup_opens_after_late_readiness_unless_silent(self):
+ for silent in (False, True):
+ with self.subTest(silent=silent):
+ manager = self.manager(ready=False)
+ manager.waiting_for_startup.return_value = True
+ manager.healthy.return_value = True
+ tray = self.tray(manager, self.preparation())
+ tray.gate.waiting = True
+ tray.gate.poll.return_value = False
+ tray.silent = silent
+ self.run_tray(tray)
+ ServiceRevivalTests.one_health_round(self, tray)
+ self.assertEqual(tray.gate.open.call_count, int(not silent))
+
+ def test_a_pending_child_that_exits_records_failure_before_revival(self):
+ manager = self.manager(ready=False)
+ manager.waiting_for_startup.return_value = True
+ manager.specs = (ServiceRevivalTests.SPEC,)
+ manager.healthy.return_value = False
+ tray = self.tray(manager, self.preparation())
+ self.run_tray(tray)
+ manager.waiting_for_startup.return_value = False
+ tray.gate.poll.return_value = False
+ ServiceRevivalTests.one_health_round(self, tray)
+ record = runtime_prepare.read_record(self.root / "state", pid=os.getpid())
+ self.assertFalse(record["ok"])
+ self.assertEqual(record["steps"][-1]["state"], "failed")
+ self.assertIsNone(tray._pending_startup)
+ tray.icon.notify.assert_called_once()
+ tray.gate.open.assert_not_called()
+
+ def test_restarting_a_slow_service_records_pending_without_a_failure_notice(self):
+ manager = self.manager(ready=False)
+ manager.waiting_for_startup.return_value = True
+ tray = self.tray(manager, self.preparation())
+ icon = self.press_restart(tray)
+ self.assertIn("仍在启动", icon.notify.call_args.args[0])
+ self.assertNotIn("未能", icon.notify.call_args.args[0])
+ self.assertFalse(runtime_prepare.read_record(self.root / "state", pid=os.getpid())["ok"])
+
+ def test_source_sync_completes_only_its_ready_pending_update_on_a_later_health_round(self):
+ for outcome in ("ready", "exited", "schema", "newer"):
+ with self.subTest(outcome=outcome):
+ manager = self.manager(ready=False)
+ manager.specs = (ServiceRevivalTests.SPEC,)
+ manager.waiting_for_startup.return_value = True
+ tray = self.tray(manager, self.preparation())
+ tray.versions.state = "updated"
+ tray.windows_updates = FakeUpdates(WindowsUpdatePreparation("services", "测试通过。"))
+ tray.gate.poll.return_value = False
+ icon = Mock()
+ tray.sync_source(icon)
+ LocalRebuildTests.drain(self, tray)
+ message = icon.notify.call_args.args[0]
+ self.assertIn("仍在启动", message)
+ self.assertNotIn("未能恢复", message)
+ self.assertIsNotNone(tray.windows_updates.pending())
+ manager.waiting_for_startup.return_value = False
+ manager.healthy.return_value = outcome != "exited"
+ if outcome == "schema":
+ manager.readiness_problems.return_value = ["HTTPS 未就绪:数据库结构与迁移不一致"]
+ if outcome == "newer":
+ tray.windows_updates.mark_pending("next-commit", ())
+ ServiceRevivalTests.one_health_round(self, tray)
+ pending = tray.windows_updates.pending()
+ if outcome == "ready":
+ self.assertIsNone(pending)
+ else:
+ self.assertIsNotNone(pending)
+ self.assertEqual(pending.commit,
+ "next-commit" if outcome == "newer" else tray.version.commit)
+ record = runtime_prepare.read_record(self.root / "state", pid=os.getpid())
+ self.assertEqual(record["ok"], outcome in ("ready", "newer"))
+
def test_restart_migrates_while_the_services_are_stopped(self):
tray = self.tray(self.manager(), self.preparation())
icon = self.press_restart(tray)