Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
ci: capture Windows fault-module evidence for native worker crashes (#8152)
Adam Eivycc77103e4968
diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.mdindex b4538c6bb4..2c139b6be9 100644--- a/docs/GITHUB_ACTIONS.md+++ b/docs/GITHUB_ACTIONS.md@@ -718,6 +718,24 @@ other canceling an older run when a newer commit arrives. Scheduled, manuallydispatched, and release-called runs skip this sibling cancellation so theiraggregate diagnostics and cache post-steps can complete normally.+### Windows native worker crash evidence++When Vitest reports a worker exit with a crashed test filename, the test+launcher queries Windows Application Error event 1000 before the existing+single-file retry, and again if that retry crashes. Each query is limited to+the failed invocation's time window, 32 events and 10 seconds. Logs and the+step summary retain at most ten Node event candidates: Node/module versions,+module basename, exception code and fault offset. Raw event messages, paths,+machine identity and memory dumps are excluded. Diagnostic failures do not+change the test result or add retries.++These events can narrow the native module involved; they do not establish+causality or identify the triggering operation. A missing or unavailable+event is explicitly inconclusive. Correlate candidates with the crashed+test and a minimal Windows reproduction before changing production metadata+handling (#8152). Microsoft documents these fields in its+[application crash troubleshooting guide](https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/troubleshoot-application-service-crashing-behavior).+### External cancellation and one automatic retryNot every cancel comes from this repository. When several PRs build at once,diff --git a/scripts/ci-windows-crash-diagnostics.js b/scripts/ci-windows-crash-diagnostics.jsnew file mode 100644index 0000000000..d156180bd7--- /dev/null+++ b/scripts/ci-windows-crash-diagnostics.js@@ -0,0 +1,60 @@+import { fileURLToPath } from 'node:url';+import { bufferedSpawn } from '../server/lib/bufferedSpawn.js';+import { writeStepSummary } from './lib/githubOutput.js';++const scriptPath = fileURLToPath(new URL('./ci-windows-crash-diagnostics.ps1', import.meta.url));+const unavailable = 'Windows native crash diagnostics unavailable (query failed or invalid response).';++// Event messages, paths, machine names and dump contents can contain private+// data. Only these bounded, validated fields may reach logs or step summaries.+function diagnosticLine(event) {+ if (event?.AppName?.toLowerCase() !== 'node.exe') throw new Error('not a Node event');+ const field = (name, pattern) => {+ const value = event[name];+ return typeof value === 'string' && pattern.test(value) ? value : 'unavailable';+ };+ return 'Windows native crash candidate (Application Error 1000): '+ + `node=${field('AppVersion', /^\d[\d.]{0,39}$/)} `+ + `module=${field('ModuleName', /^[\w+-]{1,100}\.(?:dll|exe|node)$/i)} `+ + `moduleVersion=${field('ModuleVersion', /^\d[\d.]{0,39}$/)} `+ + `exception=${field('ExceptionCode', /^(?:0x)?[a-f\d]{8}$/i)} `+ + `offset=${field('FaultingOffset', /^(?:0x)?[a-f\d]{1,16}$/i)}`;+}++/**+ * Best-effort evidence before a retry/cancel can erase the crash context.+ * Events are candidates from this invocation's time window, not proof that+ * a particular module caused the worker crash. Never changes the test result.+ */+export async function reportWindowsCrashDiagnostics(startedAt, {+ platform = process.platform, run = bufferedSpawn, logger = console, writeSummary = writeStepSummary,+} = {}) {+ if (platform !== 'win32') return { outcome: 'not-windows' };+ let lines;+ let outcome;+ try {+ const since = new Date(startedAt).toISOString();+ const until = new Date().toISOString();+ const result = await run('powershell.exe', ['-NoLogo', '-NoProfile', '-NonInteractive', '-File', scriptPath], {+ timeoutMs: 10_000,+ env: { ...process.env, PORTOS_CRASH_SINCE: since, PORTOS_CRASH_UNTIL: until },+ });+ if (!result.success) throw new Error('event query failed');+ const events = JSON.parse(result.stdout.replace(/^\uFEFF/, '').trim());+ if (!Array.isArray(events) || events.length > 10) throw new Error('invalid events');+ lines = events.map(diagnosticLine);+ outcome = lines.length ? 'candidates' : 'no-events';+ if (!lines.length) lines = ['Windows native crash diagnostics: no matching Node Application Error event in this invocation; cause remains unverified.'];+ } catch {+ // In particular, never echo PowerShell stderr or an exception's message.+ lines = [unavailable];+ outcome = 'unavailable';+ }+ for (const line of lines) logger.warn(line);+ try {+ writeSummary(lines.join('\n\n'));+ } catch {+ logger.warn('Windows native crash diagnostic summary could not be written; see the job log.');+ }+ return { outcome };+}diff --git a/scripts/ci-windows-crash-diagnostics.ps1 b/scripts/ci-windows-crash-diagnostics.ps1new file mode 100644index 0000000000..cbadde4f58--- /dev/null+++ b/scripts/ci-windows-crash-diagnostics.ps1@@ -0,0 +1,41 @@+param()+$ErrorActionPreference = 'Stop'+$ProgressPreference = 'SilentlyContinue'+[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false)++# Query structured XML, never the localized message (which includes paths,+# machine identity and report identifiers). No registry changes or dumps.+$filter = @{+ LogName = 'Application'+ ProviderName = 'Application Error'+ Id = 1000+ StartTime = [DateTimeOffset]::Parse($env:PORTOS_CRASH_SINCE).LocalDateTime+ EndTime = [DateTimeOffset]::Parse($env:PORTOS_CRASH_UNTIL).LocalDateTime+}+try {+ $events = @(Get-WinEvent -FilterHashtable $filter -MaxEvents 32 -ErrorAction Stop)+} catch {+ if ($_.FullyQualifiedErrorId -notlike 'NoMatchingEventsFound*') { throw }+ $events = @()+}+$records = @(+ foreach ($event in $events) {+ $fields = @{}+ $xml = [xml]$event.ToXml()+ foreach ($data in $xml.Event.EventData.Data) {+ $fields[[string]$data.Name] = [string]$data.'#text'+ }+ if ($fields.AppName -ieq 'node.exe') {+ # JS validates these values again before publishing any evidence.+ @{+ AppName = $fields.AppName+ AppVersion = $fields.AppVersion+ ModuleName = $fields.ModuleName+ ModuleVersion = $fields.ModuleVersion+ ExceptionCode = $fields.ExceptionCode+ FaultingOffset = $fields.FaultingOffset+ }+ }+ }+)+ConvertTo-Json -InputObject @($records | Select-Object -First 10) -Compressdiff --git a/scripts/ci-windows-crash-diagnostics.test.js b/scripts/ci-windows-crash-diagnostics.test.jsnew file mode 100644index 0000000000..336fab3aa2--- /dev/null+++ b/scripts/ci-windows-crash-diagnostics.test.js@@ -0,0 +1,94 @@+import { describe, expect, it, vi } from 'vitest';+import { bufferedSpawn } from '../server/lib/bufferedSpawn.js';+import { reportWindowsCrashDiagnostics } from './ci-windows-crash-diagnostics.js';++const event = {+ AppName: 'node.exe', AppVersion: '24.15.0', ModuleName: 'example-native.dll',+ ModuleVersion: '1.2.3.4', ExceptionCode: 'c0000409', FaultingOffset: '0000000000012345',+};++function setup(result = { success: true, stdout: JSON.stringify([event]) }) {+ return {+ platform: 'win32', run: vi.fn().mockResolvedValue(result),+ logger: { warn: vi.fn() }, writeSummary: vi.fn(),+ };+}++describe('Windows crash diagnostic publication boundary', () => {+ it('retains fault-module evidence while excluding paths, identities and command injection', async () => {+ const options = setup({ success: true, stdout: JSON.stringify([+ { ...event, AppPath: 'C:/Users/example-private/node.exe', Computer: 'example-private-host', Message: 'example-private-message' },+ { ...event, ModuleName: 'C:/Users/example-private/private.dll', ExceptionCode: '\n::error::private' },+ ]) });+ expect(await reportWindowsCrashDiagnostics(Date.now() - 1000, options)).toEqual({ outcome: 'candidates' });+ const published = options.writeSummary.mock.calls[0][0];+ expect(published).toContain('node=24.15.0 module=example-native.dll moduleVersion=1.2.3.4 exception=c0000409 offset=0000000000012345');+ expect(published).toContain('module=unavailable');+ expect(published).toContain('exception=unavailable');+ expect(published).not.toMatch(/private|::error|Users/);+ expect(options.run.mock.calls[0][2]).toMatchObject({ timeoutMs: 10_000 });+ });++ it('distinguishes an empty event window from query/schema failures without echoing stderr', async () => {+ const empty = setup({ success: true, stdout: '[]' });+ expect(await reportWindowsCrashDiagnostics(Date.now(), empty)).toEqual({ outcome: 'no-events' });+ expect(empty.writeSummary.mock.calls[0][0]).toContain('cause remains unverified');+ for (const result of [+ { success: false, timedOut: true, stdout: '[]', stderr: 'example-private-error' },+ { success: true, stdout: '' },+ { success: true, stdout: '{}' },+ { success: true, stdout: JSON.stringify([{ AppName: 42 }]) },+ { success: true, stdout: JSON.stringify(Array(11).fill(event)) },+ ]) {+ const options = setup(result);+ expect(await reportWindowsCrashDiagnostics(Date.now(), options)).toEqual({ outcome: 'unavailable' });+ expect(options.writeSummary.mock.calls[0][0]).toContain('diagnostics unavailable');+ expect(options.writeSummary.mock.calls[0][0]).not.toContain('example-private');+ }+ });++ it('does not turn a missing summary file into a failed test run, or query other platforms', async () => {+ const options = setup();+ options.writeSummary.mockImplementation(() => { throw Error('example-private-path'); });+ expect(await reportWindowsCrashDiagnostics(Date.now(), options)).toEqual({ outcome: 'candidates' });+ expect(options.logger.warn.mock.calls.flat().join('\n')).not.toContain('example-private');+ options.run.mockClear();+ expect(await reportWindowsCrashDiagnostics(Date.now(), { ...options, platform: 'linux' })).toEqual({ outcome: 'not-windows' });+ expect(options.run).not.toHaveBeenCalled();+ });+});++describe.runIf(process.platform === 'win32')('real Windows PowerShell diagnostic query', () => {+ it('projects named event XML fields without publishing raw event content', async () => {+ const options = setup();+ options.run = async (command, args, spawnOptions) => {+ const path = args.at(-1).replace(/'/g, "''");+ const result = await bufferedSpawn(command, ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', `+ function Get-WinEvent {+ param($FilterHashtable, $MaxEvents, $ErrorAction)+ if ($FilterHashtable.Id -ne 1000 -or $MaxEvents -ne 32) { throw 'wrong query' }+ $item = New-Object PSObject+ $item | Add-Member -MemberType ScriptMethod -Name ToXml -Value {+ '<Event><EventData><Data Name="AppName">node.exe</Data><Data Name="AppVersion">24.15.0</Data><Data Name="ModuleName">example-native.dll</Data><Data Name="ModuleVersion">1.2.3.4</Data><Data Name="ExceptionCode">c0000409</Data><Data Name="FaultingOffset">0000000000012345</Data><Data Name="AppPath">example-private-path</Data></EventData></Event>'+ }+ $item+ }+ & '${path}'+ `], spawnOptions);+ expect(result.success, result.stderr).toBe(true);+ expect(result.stdout).not.toContain('example-private');+ return result;+ };+ expect(await reportWindowsCrashDiagnostics(Date.now() - 1000, options)).toEqual({ outcome: 'candidates' });+ expect(options.writeSummary.mock.calls[0][0]).toContain('module=example-native.dll');+ });++ it('can query the real Application Error provider with a bounded empty future window', async () => {+ const options = setup();+ options.run = (command, args, spawnOptions) => bufferedSpawn(command, args, {+ ...spawnOptions,+ env: { ...spawnOptions.env, PORTOS_CRASH_UNTIL: new Date(Date.now() + 120_000).toISOString() },+ });+ expect(await reportWindowsCrashDiagnostics(Date.now() + 60_000, options)).toEqual({ outcome: 'no-events' });+ });+});diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.jsindex dbc4512fde..057bc9fe16 100644--- a/scripts/run-ci-tests.js+++ b/scripts/run-ci-tests.js@@ -10,6 +10,7 @@ import { killProcessTree, prepareCliSpawn } from '../server/lib/bufferedSpawn.jsimport { isDirectlyInvoked } from './lib/directInvocation.js';import { sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';import { writeStepEnv, writeStepSummary } from './lib/githubOutput.js';+import { reportWindowsCrashDiagnostics } from './ci-windows-crash-diagnostics.js';const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..');@@ -264,10 +265,12 @@ function runNpmWithReport(scope, script, extraArgs, reportPath) {});}-async function spawnNpm(scope, script, extraArgs, label) {+export async function spawnNpm(scope, script, extraArgs, label, {+ run = runNpm, diagnose = reportWindowsCrashDiagnostics,+} = {}) {console.log(`Running ${scope} ${label}${extraArgs.length ? ` (${extraArgs.length} selector argument(s))` : ''}.`);const startedAt = Date.now();- const result = await runNpm(scope, script, extraArgs);+ const result = await run(scope, script, extraArgs);recordVitestDuration(scope, label, startedAt);if (result.error) {console.error(result.error.message);@@ -276,6 +279,8 @@ async function spawnNpm(scope, script, extraArgs, label) {if (result.status === 0) return 0;if (result.interrupted) return result.status;+ if (extractCrashedTestFile(result.output)) await diagnose(startedAt);+const plan = planCrashRetry(scope, result.output);if (!plan.retry) {if (plan.crashedPath) writeStepEnv('CI_CRASHED_TEST_FILE', plan.crashedPath);@@ -289,7 +294,7 @@ async function spawnNpm(scope, script, extraArgs, label) {// reproduces or was a one-off runner fault.console.warn(`⚠️ Vitest worker crashed natively on ${relPath} — retrying that file once`);const retryStartedAt = Date.now();- const retry = await runNpm(scope, script, [selector]);+ const retry = await run(scope, script, [selector]);recordVitestDuration(scope, `crash retry: ${relPath}`, retryStartedAt);if (!retry.error && retry.status === 0) {if (hasOtherTestFailures(result.output)) {@@ -303,6 +308,7 @@ async function spawnNpm(scope, script, extraArgs, label) {}if (extractCrashedTestFile(retry.output)) {+ await diagnose(retryStartedAt);console.error(`❌ ${relPath} crashed again on retry — this reproduces, not a runner fluke`);writeStepEnv('CI_CRASHED_TEST_FILE', relPath);} else {diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.jsindex 119e8a196e..86fd21ac00 100644--- a/scripts/run-ci-tests.test.js+++ b/scripts/run-ci-tests.test.js@@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync,import { tmpdir } from 'os';import { join } from 'path';import { stripVTControlCharacters } from 'node:util';-import { afterEach, describe, expect, it } from 'vitest';+import { afterEach, describe, expect, it, vi } from 'vitest';import {structuredFailureDiagnostics,@@ -15,6 +15,7 @@ import {repoRelativeFromCrashPath,requiresSourceFiles,shardArgs,+ spawnNpm,toRunnerPath,} from './run-ci-tests.js';import { killProcessTree } from '../server/lib/bufferedSpawn.js';@@ -279,6 +280,45 @@ describe('relatedInputs', () => {});});+describe('native crash diagnostics before retry/cancellation (#8152)', () => {+ const output = 'Worker exited unexpectedly with exit code 3221226505 during started state while running test file '+ + 'D:/a/PortOS/PortOS/server/services/sprites/walk.test.js\n'+ + 'Test Files 798 passed | 2 skipped (801)\nTests 15077 passed | 61 skipped (15145)\nErrors 1 error';++ it('captures each failed invocation before returning nonzero with the same one-retry ceiling', async () => {+ const order = [];+ const run = vi.fn(async () => {+ order.push('run');+ return { status: 1, output };+ });+ const diagnose = vi.fn(async () => { order.push('diagnose'); });+ expect(await spawnNpm('server', 'test:ci', ['--shard=3/3'], 'fixture', { run, diagnose })).toBe(1);+ expect(order).toEqual(['run', 'diagnose', 'run', 'diagnose']);+ expect(run.mock.calls[1]).toEqual(['server', 'test:ci', ['./services/sprites/walk.test.js']]);+ });++ it('retains diagnostic evidence for recovered crashes without clearing unrelated assertion failures', async () => {+ for (const [originalOutput, expected] of [[output, 0], [output.replace('798 passed', '797 passed | 1 failed'), 1]]) {+ const run = vi.fn().mockResolvedValueOnce({ status: 1, output: originalOutput })+ .mockResolvedValueOnce({ status: 0, output: 'passed' });+ const diagnose = vi.fn();+ expect(await spawnNpm('server', 'test:ci', [], 'fixture', { run, diagnose })).toBe(expected);+ expect(diagnose).toHaveBeenCalledTimes(1);+ expect(run).toHaveBeenCalledTimes(2);+ }+ });++ it('leaves assertion failures and interrupted runs outside native diagnostics/retry', async () => {+ for (const result of [{ status: 1, output: 'FAIL assertion' }, { status: 143, output, interrupted: 'SIGTERM' }]) {+ const run = vi.fn().mockResolvedValue(result);+ const diagnose = vi.fn();+ expect(await spawnNpm('server', 'test:ci', [], 'fixture', { run, diagnose })).toBe(result.status);+ expect(run).toHaveBeenCalledTimes(1);+ expect(diagnose).not.toHaveBeenCalled();+ }+ });+});+describe('structured failure diagnostics', () => {it('reports a run-level rejection and worker cause when every test passed (#11244)', () => {