Skip to content

The agent that grows with you

fix(desktop): the state.db pre-flight scales its deadline with the database and keeps the window responsive

Teknium64e38130c7c6
diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts
index cf12dc19695..42d569df916 100644
--- a/apps/desktop/electron/main.ts
+++ b/apps/desktop/electron/main.ts
@@ -3660,7 +3660,7 @@ function resolveCheckoutUpdateStrategy(): CheckoutStrategy {
throw new Error(message)
}
- preflightStateDb({
+ await preflightStateDb({
python: managed ? null : await findPythonForRoot(root),
launcher,
script: path.join(root, 'hermes_cli', 'backup_sqlite.py'),
diff --git a/apps/desktop/electron/updater/state-db-preflight.test.ts b/apps/desktop/electron/updater/state-db-preflight.test.ts
index 5dea73ecfc0..6ef0f2931a7 100644
--- a/apps/desktop/electron/updater/state-db-preflight.test.ts
+++ b/apps/desktop/electron/updater/state-db-preflight.test.ts
@@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url'
import { test } from 'vitest'
-import { preflightStateDb } from './state-db-preflight'
+import { preflightStateDb, stateDbPreflightTimeoutMs } from './state-db-preflight'
test('the desktop preflight publishes committed WAL rows before its caller can stop the backend', async (): Promise<void> => {
const home: string = fs.mkdtempSync(path.join(os.tmpdir(), 'desktop-db-'))
@@ -45,7 +45,7 @@ c.close()
try {
await once(child.stdout!, 'data')
- preflightStateDb({
+ await preflightStateDb({
python,
script,
home,
@@ -87,7 +87,7 @@ with sqlite3.connect(sys.argv[1]) as c:
}
})
-test('a managed installation runs the snapshot through the installation launcher', (): void => {
+test('a managed installation runs the snapshot through the installation launcher', async (): Promise<void> => {
const home: string = fs.mkdtempSync(path.join(os.tmpdir(), 'managed-preflight-'))
const shims: string = fs.mkdtempSync(path.join(os.tmpdir(), 'launcher-shim-'))
const python: string = process.env.HERMES_PYTHON || 'python3'
@@ -125,7 +125,7 @@ test('a managed installation runs the snapshot through the installation launcher
assert.equal(created.status, 0, created.stderr)
- preflightStateDb({
+ await preflightStateDb({
python: null,
launcher: shim,
script,
@@ -143,13 +143,13 @@ test('a managed installation runs the snapshot through the installation launcher
}
})
-test('an older selected checkout without the snapshot helper refuses before backend stop', (): void => {
+test('an older selected checkout without the snapshot helper refuses before backend stop', async (): Promise<void> => {
const oldRoot: string = fs.mkdtempSync(path.join(os.tmpdir(), 'old-preflight-'))
let stopped = false
try {
- assert.throws((): void => {
- preflightStateDb({
+ await assert.rejects(async (): Promise<void> => {
+ await preflightStateDb({
python: process.env.HERMES_PYTHON || 'python3',
script: path.join(oldRoot, 'hermes_cli', 'backup_sqlite.py'),
home: oldRoot,
@@ -162,3 +162,51 @@ test('an older selected checkout without the snapshot helper refuses before back
fs.rmSync(oldRoot, { recursive: true, force: true })
}
})
+
+test('the deadline grows with the main database and WAL, but stays bounded', (): void => {
+ const home: string = fs.mkdtempSync(path.join(os.tmpdir(), 'preflight-budget-'))
+
+ try {
+ const floor = stateDbPreflightTimeoutMs(home)
+
+ for (const name of ['state.db', 'state.db-wal']) {
+ fs.writeFileSync(path.join(home, name), '')
+ fs.truncateSync(path.join(home, name), 2 * 1024 ** 3)
+ }
+
+ const withWal = stateDbPreflightTimeoutMs(home)
+ fs.unlinkSync(path.join(home, 'state.db-wal'))
+ const withoutWal = stateDbPreflightTimeoutMs(home)
+ assert.ok(floor >= 5 * 60_000)
+ assert.ok(withWal > withoutWal && withoutWal > floor)
+ fs.truncateSync(path.join(home, 'state.db'), 100 * 1024 ** 3)
+ assert.ok(stateDbPreflightTimeoutMs(home) <= 30 * 60_000)
+ } finally {
+ fs.rmSync(home, { recursive: true, force: true })
+ }
+})
+
+test('the event loop stays available while a real snapshot process is running', async (): Promise<void> => {
+ const home: string = fs.mkdtempSync(path.join(os.tmpdir(), 'preflight-responsive-'))
+ const script = path.join(home, 'snapshot.py')
+ fs.writeFileSync(
+ script,
+ `import pathlib, sys, time
+home = pathlib.Path(sys.argv[1])
+deadline = time.monotonic() + 3
+while not (home / 'release').exists():
+ if time.monotonic() > deadline:
+ raise RuntimeError('parent event loop was blocked')
+ time.sleep(0.01)
+print('snapshot finished')
+`
+ )
+ const timer = setTimeout(() => fs.writeFileSync(path.join(home, 'release'), ''), 100)
+
+ try {
+ await preflightStateDb({ python: process.env.HERMES_PYTHON || 'python3', script, home, log: (): void => {} })
+ } finally {
+ clearTimeout(timer)
+ fs.rmSync(home, { recursive: true, force: true })
+ }
+})
diff --git a/apps/desktop/electron/updater/state-db-preflight.ts b/apps/desktop/electron/updater/state-db-preflight.ts
index 2eb96d4e833..f2775ad1bb1 100644
--- a/apps/desktop/electron/updater/state-db-preflight.ts
+++ b/apps/desktop/electron/updater/state-db-preflight.ts
@@ -1,7 +1,33 @@
-import { execFileSync } from 'node:child_process'
+import { execFile } from 'node:child_process'
+import fs from 'node:fs'
+import path from 'node:path'
+import { promisify } from 'node:util'
import { hiddenWindowsChildOptions } from '../windows-child-options'
+const DEFAULT_TIMEOUT_MS = 5 * 60_000
+const MAX_TIMEOUT_MS = 30 * 60_000
+const execFileAsync = promisify(execFile)
+const COPY_RATE_BYTES_PER_SECOND = 5 * 1024 * 1024
+const TIMEOUT_SLACK_MS = 60_000
+
+export function stateDbPreflightTimeoutMs(home: string): number {
+ let bytes = 0
+
+ for (const name of ['state.db', 'state.db-wal']) {
+ try {
+ bytes += fs.statSync(path.join(home, name)).size
+ } catch {
+ // Missing WAL is normal; an unreadable size retains the conservative floor.
+ }
+ }
+
+ return Math.min(
+ MAX_TIMEOUT_MS,
+ Math.max(DEFAULT_TIMEOUT_MS, Math.ceil(bytes / COPY_RATE_BYTES_PER_SECOND) * 1_000 + TIMEOUT_SLACK_MS)
+ )
+}
+
interface StateDbPreflight {
python: string | null
script: string
@@ -16,8 +42,14 @@ interface StateDbPreflight {
launcher?: string | null
}
-// Synchronous by design: the caller must not stop the backend before the snapshot.
-export function preflightStateDb({ python, script, home, log, launcher = null }: StateDbPreflight): void {
+// Await completion before stopping the backend, while keeping Electron responsive.
+export async function preflightStateDb({
+ python,
+ script,
+ home,
+ log,
+ launcher = null
+}: StateDbPreflight): Promise<void> {
try {
const command: string | null = launcher ?? python
@@ -37,20 +69,19 @@ export function preflightStateDb({ python, script, home, log, launcher = null }:
throw new Error('The pre-flight snapshot contains an unsafe Windows command argument.')
}
- const result: string = execFileSync(
+ const { stdout } = await execFileAsync(
viaCmd ? (process.env.ComSpec ?? 'cmd.exe') : command,
viaCmd
? ['/d', '/v:off', '/s', '/c', `""${command}" ${args.map((arg: string): string => `"${arg}"`).join(' ')}"`]
: args,
hiddenWindowsChildOptions({
encoding: 'utf8',
- timeout: 30_000,
- stdio: ['ignore', 'pipe', 'pipe'],
+ timeout: stateDbPreflightTimeoutMs(home),
windowsVerbatimArguments: viaCmd
})
)
- log(`[updates] state.db pre-flight: ${result.trim()}`)
+ log(`[updates] state.db pre-flight: ${stdout.trim()}`)
} catch (error: unknown) {
const message =
`state.db pre-flight failed: ${error instanceof Error ? error.message : String(error)}. ` +
diff --git a/hermes_cli/backup_sqlite.py b/hermes_cli/backup_sqlite.py
index 9f5f0054036..7ae9a755c4f 100644
--- a/hermes_cli/backup_sqlite.py
+++ b/hermes_cli/backup_sqlite.py
@@ -6,6 +6,7 @@ imports cannot load. Full and quick backups use the same SQLite copy operation.
import json
import logging
import os
+import signal
import sqlite3
import sys
import tempfile
@@ -82,12 +83,27 @@ def _safe_copy_db(src: Path, dst: Path, *, timeout_seconds: float = 10.0) -> boo
_close_quietly(conn)
+# Longer than the Desktop's 30-minute pre-flight cap: a staging file this old
+# has no live writer. A hard-killed pre-flight (Windows TerminateProcess)
+# cannot unlink its own staging copy, and each one is a full-size database.
+_ABANDONED_STAGING_SECONDS = 2 * 3600
+
+
+def _reclaim_abandoned_staging(home: Path, prefix: str) -> None:
+ horizon = time.time() - _ABANDONED_STAGING_SECONDS
+ for staged in home.glob(f"{prefix}*.partial*"): # + the copy's -journal
+ with suppress(OSError):
+ if staged.stat().st_mtime < horizon:
+ staged.unlink()
+
+
def preflight_state_db(home: Path) -> dict:
"""Publish an emergency snapshot; do not prune recovery files on failure."""
source = home / "state.db"
if not source.exists():
return {"path": None, "message": "state.db not found (fresh install?)"}
prefix = "state.db.pre-update-emergency-"
+ _reclaim_abandoned_staging(home, prefix)
stamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S-%fZ")
destination = home / f"{prefix}{stamp}-{os.getpid()}.bak"
fd, name = tempfile.mkstemp(prefix=prefix, suffix=".partial", dir=home)
@@ -116,4 +132,7 @@ def preflight_state_db(home: Path) -> dict:
if __name__ == "__main__":
+ # The Desktop stops a pre-flight past its deadline with SIGTERM; exit through
+ # the staging cleanup instead of dying with a full-size .partial left behind.
+ signal.signal(signal.SIGTERM, lambda signum, _frame: sys.exit(128 + signum))
print(json.dumps(preflight_state_db(Path(sys.argv[1]))))
diff --git a/tests/hermes_cli/test_backup_preflight_staging.py b/tests/hermes_cli/test_backup_preflight_staging.py
new file mode 100644
index 00000000000..c94808c26d5
--- /dev/null
+++ b/tests/hermes_cli/test_backup_preflight_staging.py
@@ -0,0 +1,61 @@
+"""A stopped or killed update pre-flight must not leave full-size staging copies behind.
+
+Desktop stops a pre-flight that outlives its deadline; each abandoned
+``state.db.pre-update-emergency-*.partial`` is a whole copy of state.db, and
+reporters accumulated 11.7 GB of them (#124972, #128605).
+"""
+import os
+import signal
+import sqlite3
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+SCRIPT = Path(__file__).resolve().parents[2] / "hermes_cli" / "backup_sqlite.py"
+
+
+def _home_with_db(tmp_path: Path, rows: int) -> Path:
+ home = tmp_path / "home"
+ home.mkdir()
+ with sqlite3.connect(home / "state.db") as db:
+ db.execute("PRAGMA journal_mode=WAL")
+ db.execute("CREATE TABLE messages (body TEXT)")
+ db.executemany("INSERT INTO messages VALUES (?)", [("x" * 4000,)] * rows)
+ return home
+
+
+def _staging(home: Path) -> list[str]:
+ return sorted(p.name for p in home.glob("state.db.pre-update-emergency-*.partial*"))
+
+
+def test_sigterm_mid_copy_removes_the_staging_copy(tmp_path):
+ home = _home_with_db(tmp_path, 60000)
+ proc = subprocess.Popen([sys.executable, "-I", "-S", str(SCRIPT), str(home)],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ deadline = time.monotonic() + 30
+ while not _staging(home) and proc.poll() is None and time.monotonic() < deadline:
+ time.sleep(0.005)
+ proc.send_signal(signal.SIGTERM)
+ proc.communicate(timeout=60)
+
+ assert _staging(home) == []
+
+
+def test_a_new_preflight_reclaims_staging_abandoned_by_a_killed_one(tmp_path):
+ home = _home_with_db(tmp_path, 10)
+ abandoned = home / "state.db.pre-update-emergency-killed.partial"
+ abandoned.write_bytes(b"x" * 4096)
+ hours_ago = time.time() - 3 * 3600
+ os.utime(abandoned, (hours_ago, hours_ago))
+ in_flight = home / "state.db.pre-update-emergency-running.partial"
+ in_flight.write_bytes(b"x" * 4096)
+
+ result = subprocess.run([sys.executable, "-I", "-S", str(SCRIPT), str(home)],
+ capture_output=True, text=True, timeout=60, check=False)
+
+ assert result.returncode == 0, result.stderr
+ assert _staging(home) == [in_flight.name]