Skip to content

The agent that grows with you

fix(desktop-update): serialize hand-off log appends across concurrent writers

liuhao10240b62510eb34a
diff --git a/scripts/desktop-update/windows.ps1 b/scripts/desktop-update/windows.ps1
index ddf5323f046..03e91bb4ddc 100644
--- a/scripts/desktop-update/windows.ps1
+++ b/scripts/desktop-update/windows.ps1
@@ -72,6 +72,7 @@ param(
[int]$ProbeTimeoutSeconds = 60,
[switch]$SelfTestUi,
[switch]$SelfTestPipeDrain,
+ [switch]$SelfTestLogConcurrency,
[switch]$SelfTestMarker,
[switch]$SelfTestWorkingDirectory,
[string]$HandoffRun = "",
@@ -89,7 +90,7 @@ if ($PSBoundParameters.ContainsKey("Branch") -and $PSBoundParameters.ContainsKey
}
$targetArgs = if ($Channel) { @("--channel", $Channel.ToLowerInvariant()) } else { @("--branch", $Branch) }
-if (-not $SelfTestUi -and -not $SelfTestPipeDrain -and -not $InstallRoot) {
+if (-not $SelfTestUi -and -not $SelfTestPipeDrain -and -not $SelfTestLogConcurrency -and -not $InstallRoot) {
# Mandatory in spirit; relaxed in the signature only so the self-test
# switches can drive the UI / the pipe drain without a checkout.
throw "-InstallRoot is required"
@@ -103,10 +104,45 @@ $MarkerPath = Join-Path $HermesHome ".hermes-update-in-progress"
$LogDir = Join-Path $HermesHome "logs"
$LogPath = Join-Path $LogDir "desktop-update-handoff.log"
$ResultPath = Join-Path $HermesHome ".hermes-update-result.json"
+$script:HandoffLogMutex = $null
+# One mutex per log path (case-normalized: the same HERMES_HOME spelled
+# differently must still serialize), hashed because a mutex name cannot carry
+# path characters.
+$handoffLogHash = ""
+$handoffSha = [System.Security.Cryptography.SHA1]::Create()
+try {
+ $handoffLogHash = [BitConverter]::ToString($handoffSha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($LogPath.ToLowerInvariant())), 0, 8).Replace("-", "")
+} finally { $handoffSha.Dispose() }
+$script:HandoffLogMutexName = "Global\HermesHandoffLog-$handoffLogHash"
function Write-HandoffLog([string]$Message) {
$line = "{0:yyyy-MM-ddTHH:mm:ssK} {1}" -f (Get-Date), $Message
- try { Add-Content -LiteralPath $LogPath -Value $line -Encoding UTF8 } catch {}
+ # Add-Content is unsafe once a second writer is live: PowerShell caches a
+ # content writer per path, so the second writer (an overlapping hand-off
+ # process, or any future runspace writer) gets a handle the first already
+ # consumed -- Windows PowerShell 5.1 floods "stream is not readable"
+ # errors while PowerShell 7 silently loses lines (#126152). A named mutex
+ # plus one FileStream per write serializes appends across processes and
+ # runspaces; the bounded wait and the empty catch keep the "logging must
+ # never break the update" contract.
+ $held = $false
+ try {
+ $mutex = $script:HandoffLogMutex
+ if (-not $mutex) {
+ # Named, so every writer's own instance joins the same OS object;
+ # lazily created because runspaces cannot share the parent's
+ # instance. Created unsignalled: ownership comes from WaitOne.
+ $mutex = $script:HandoffLogMutex = New-Object System.Threading.Mutex($false, $script:HandoffLogMutexName)
+ }
+ try { $held = $mutex.WaitOne(5000) }
+ catch [System.Threading.AbandonedMutexException] { $held = $true } # a writer died mid-append; the wait still granted ownership
+ $bytes = [System.Text.Encoding]::UTF8.GetBytes($line + [Environment]::NewLine)
+ $fs = New-Object System.IO.FileStream($LogPath, [System.IO.FileMode]::Append, [System.IO.FileAccess]::Write, [System.IO.FileShare]::ReadWrite)
+ try { $fs.Write($bytes, 0, $bytes.Length); $fs.Flush() } finally { $fs.Dispose() }
+ } catch {
+ } finally {
+ if ($held) { try { $script:HandoffLogMutex.ReleaseMutex() } catch {} }
+ }
if ($MarkerOp) { return } # a helper op's stdout is its one verdict line
if ($script:ConsoleInput -and [HermesHandoff.ConsoleInput]::Selecting()) { return }
Write-Host $line
@@ -1537,6 +1573,83 @@ exit 3
exit 0
}
+# ── -SelfTestLogConcurrency: Write-HandoffLog under concurrent writers ─────
+# Add-Content drops lines (and on 5.1, errors) once a second writer is live
+# (#126152). Run the REAL function from a runspace pool -- every writer then
+# holds its own lazy mutex instance against the same named OS mutex, which is
+# exactly what an overlapping hand-off process or future runspace writer does
+# -- and hold the file to the exact writers*lines count with no torn lines.
+if ($SelfTestLogConcurrency) {
+ $writers = 8
+ $linesEach = 100
+ $target = Join-Path $TempDir ("handoff-log-concurrency-{0}.log" -f $PID)
+ Remove-Item -LiteralPath $target -Force -ErrorAction SilentlyContinue
+
+ # Single-quoted halves on purpose: the function text carries $-tokens that
+ # must survive verbatim into the runspace script. Extracted from this
+ # file's AST because ${function:...} stringifies to the body only, which
+ # would leave the runspace without a callable definition.
+ $fnAst = [System.Management.Automation.Language.Parser]::ParseFile($PSCommandPath, [ref]$null, [ref]$null).FindAll({
+ param($node)
+ $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq "Write-HandoffLog"
+ }, $true)
+ if (-not $fnAst -or $fnAst.Count -eq 0) {
+ Write-Host "LOG-CONCURRENCY SELF-TEST: FAIL -- Write-HandoffLog definition not found in this script"
+ exit 1
+ }
+ $fnText = $fnAst[0].Extent.Text
+ $body = @'
+param([string]$TargetLog, [string]$MutexName, [int]$Count, [int]$WriterId)
+$LogPath = $TargetLog
+$script:HandoffLogMutexName = $MutexName
+$script:HandoffLogMutex = $null
+$script:ConsoleInput = $false
+
+'@ + $fnText + @'
+
+for ($i = 0; $i -lt $Count; $i++) { Write-HandoffLog ("w$WriterId line$i") }
+'@
+ $pool = [runspacefactory]::CreateRunspacePool(1, $writers)
+ $pool.Open()
+ $handles = @()
+ try {
+ foreach ($w in 1..$writers) {
+ $ps = [powershell]::Create()
+ $ps.RunspacePool = $pool
+ [void]$ps.AddScript($body).AddArgument($target).AddArgument($script:HandoffLogMutexName).AddArgument($linesEach).AddArgument($w)
+ $handles += @{ PS = $ps; Handle = $ps.BeginInvoke() }
+ }
+ foreach ($h in $handles) {
+ try { $h.PS.EndInvoke($h.Handle) } catch {}
+ $h.PS.Dispose()
+ }
+ } finally {
+ $pool.Close()
+ $pool.Dispose()
+ }
+
+ $problems = @()
+ $lines = @()
+ if (Test-Path -LiteralPath $target) { $lines = @(Get-Content -LiteralPath $target) }
+ if ($lines.Count -ne ($writers * $linesEach)) {
+ $problems += "expected $($writers * $linesEach) lines, found $($lines.Count) -- concurrent writers lost or duplicated lines"
+ }
+ foreach ($w in 1..$writers) {
+ $mine = @($lines | Where-Object { $_ -match " w$w line\d+$" })
+ if ($mine.Count -ne $linesEach) { $problems += "writer $w contributed $($mine.Count) of $linesEach lines" }
+ }
+ # A torn interleave leaves a line without its timestamp prefix + payload shape.
+ $intact = @($lines | Where-Object { $_ -cmatch "^\d{4}-\d{2}-\d{2}T\S+ w\d+ line\d+$" })
+ if ($intact.Count -ne $lines.Count) { $problems += "$($lines.Count - $intact.Count) lines are torn or malformed" }
+
+ if ($problems.Count -gt 0) {
+ Write-Host "LOG-CONCURRENCY SELF-TEST: FAIL -- $($problems -join '; ')"
+ exit 1
+ }
+ Write-Host "LOG-CONCURRENCY SELF-TEST: PASS writers=$writers linesEach=$linesEach file=$target"
+ exit 0
+}
+
$savedConsoleInputMode = if ($script:ConsoleInput) { [HermesHandoff.ConsoleInput]::DisableQuickEdit() } else { $null }
try {
# -- 0. The marker was claimed before anything else (top of script) -----
diff --git a/tests/scripts/desktop_update/test_desktop_update_windows_log_concurrency.py b/tests/scripts/desktop_update/test_desktop_update_windows_log_concurrency.py
new file mode 100644
index 00000000000..c1f43074fd3
--- /dev/null
+++ b/tests/scripts/desktop_update/test_desktop_update_windows_log_concurrency.py
@@ -0,0 +1,89 @@
+"""Regression: concurrent writers must not lose hand-off log lines.
+
+``scripts/desktop-update/windows.ps1`` used to append to
+``desktop-update-handoff.log`` with ``Add-Content``. That cmdlet is unsafe
+once a second writer is live: PowerShell caches a content writer per path, so
+the second writer receives a handle the first one already consumed. Measured
+on the shape this fixture reproduces (8 writers x 100 lines, #126152):
+
+* Windows PowerShell 5.1 -- a ``GetContentWriterArgumentError`` ("stream is
+ not readable") error flood;
+* PowerShell 7 -- no errors and **58% of lines silently missing**.
+
+The hand-off log is the only record of why a Desktop update failed, and a
+second writer is a real event, not a hypothesis: two hand-off processes can
+overlap (the marker claim is a plain write with no mutual exclusion, and its
+failure path logs a warning and continues), and every future runspace writer
+joins the same race. ``Write-HandoffLog`` now serializes appends behind a
+named, per-log-path mutex and one ``FileStream`` per write.
+
+The fixture runs the REAL ``Write-HandoffLog`` (fetched as function text and
+re-homed into each runspace with its own lazily created mutex instance, the
+same way an overlapping process joins the named mutex) from a runspace pool,
+then holds the file to the exact line count, per-writer count, and intact
+line shape. It is ``platforms("windows")`` because Linux CI cannot execute
+the PowerShell hand-off.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+WINDOWS_PS1 = REPO_ROOT / "scripts" / "desktop-update" / "windows.ps1"
+
+
+def test_concurrent_writers_lose_no_handoff_log_lines(tmp_path: Path) -> None:
+ """Drive 8 concurrent runspaces through the real Write-HandoffLog.
+
+ A pass means all 800 lines landed: the total matches, each writer's 100
+ lines match, and no line lost its timestamp-prefix + payload shape (the
+ signature of a torn interleave). Under the pre-fix ``Add-Content`` the
+ same run drops lines on PowerShell 7 and errors on 5.1.
+ """
+ system_root = Path(os.environ.get("SystemRoot", r"C:\Windows"))
+ powershell = (
+ system_root / "System32" / "WindowsPowerShell" / "v1.0" / "powershell.exe"
+ )
+ if not powershell.is_file():
+ pytest.skip(f"Windows PowerShell not found at {powershell}")
+
+ env = dict(os.environ)
+ # The fixture writes its target log under TEMP; point that at tmp_path
+ # so the test leaves nothing behind.
+ env["TEMP"] = os.fspath(tmp_path)
+ env["TMP"] = os.fspath(tmp_path)
+
+ argv = [
+ os.fspath(powershell),
+ "-NoProfile",
+ "-ExecutionPolicy",
+ "Bypass",
+ "-File",
+ os.fspath(WINDOWS_PS1),
+ "-SelfTestLogConcurrency",
+ ]
+ result = subprocess.run(
+ argv,
+ capture_output=True,
+ text=True,
+ timeout=300,
+ env=env,
+ cwd=os.fspath(REPO_ROOT),
+ )
+
+ (tmp_path / "log-concurrency.stdout.log").write_text(
+ result.stdout, encoding="utf-8"
+ )
+ (tmp_path / "log-concurrency.stderr.log").write_text(
+ result.stderr, encoding="utf-8"
+ )
+ diagnosis = result.stdout[-6000:] + result.stderr[-6000:]
+ if "LOG-CONCURRENCY SELF-TEST: PASS" not in result.stdout or result.returncode != 0:
+ pytest.fail(diagnosis, pytrace=False)