Skip to content

armgit / cra

Public Imported from GitHub
0

The open-source Arc replacement for macOS. Spaces, pinned tabs, split view, Little Windows and Arc's shortcuts on Chromium, with your Arc sidebar imported in one go.

Performance recorder: one pass per wake, freeze counts that say so, one clock, stacks that arrive (#135) (#145)

Victor Kildahle4f18c556358
diff --git a/Packages/CraCore/Sources/CraCore/Perf/PerfInsights.swift b/Packages/CraCore/Sources/CraCore/Perf/PerfInsights.swift
index 5e16366..b92db6b 100644
--- a/Packages/CraCore/Sources/CraCore/Perf/PerfInsights.swift
+++ b/Packages/CraCore/Sources/CraCore/Perf/PerfInsights.swift
@@ -24,6 +24,12 @@ public enum PerfInsights {
public static let largeFootprintBytes: Int64 = 8 << 30
/// Short stalls per hour of the session that are worth a line.
public static let frequentStallsPerHour = 30.0
+ /// What recording may cost, in percent of one core (docs/performance.md).
+ public static let selfCostBudgetPct = 0.1
+ /// Cra's average CPU, percent of one core, while nobody used the Mac, over at least
+ /// `idleMinSamples` 5-second samples, that is worth a line.
+ public static let idleCpuPct = 10.0
+ public static let idleMinSamples = 12
public static func evaluate(_ r: PerfReport) -> [PerfInsight] {
var out: [PerfInsight] = []
@@ -32,15 +38,31 @@ public enum PerfInsights {
detail: "Cra \(p.craVersion) crashed, was forced to quit or the Mac lost power. Its last minute is in the export."))
}
let m = r.mainThread
- if m.longStalls > 0 {
+ if m.longFreezes > 0 {
out.append(PerfInsight(id: "freezes", severity: .warning,
- title: "Cra froze \(m.longStalls) time\(m.longStalls == 1 ? "" : "s") for a second or more",
- detail: "The longest lasted \(Self.duration(m.longestMs))\(m.hangs > 0 ? "; \(m.hangs) went past 2 s" : "")."))
+ title: "Cra froze \(m.longFreezes) time\(m.longFreezes == 1 ? "" : "s") for a second or more",
+ detail: "The longest lasted \(Self.duration(m.longestFreezeMs))\(m.hangs > 0 ? "; \(m.hangs) went past 2 s" : "")."))
}
- let hours = max(r.session.uptimeSeconds / 3600, 1.0 / 12)
- if m.stalls >= 5, Double(m.stalls) / hours >= frequentStallsPerHour {
+ let hours = max(r.session.awakeSeconds / 3600, 1.0 / 12)
+ if m.freezes >= 5, Double(m.freezes) / hours >= frequentStallsPerHour {
out.append(PerfInsight(id: "stalls", severity: .info, title: "Frequent short stalls",
- detail: "\(m.stalls) pauses of a quarter second or more since launch."))
+ detail: "\(m.freezes) pauses of a quarter second or more since launch."))
+ }
+ if m.outsideFreezes > 0 {
+ let longest = m.longestPassMs > m.longestFreezeMs ? " The longest lasted \(Self.duration(m.longestPassMs))\(m.longestPassMode.map { " (\($0))" } ?? "")." : ""
+ out.append(PerfInsight(id: "outside-freezes", severity: .info,
+ title: "\(m.outsideFreezes) long pass\(m.outsideFreezes == 1 ? "" : "es") in menus, drags, alerts or start-up",
+ detail: "Not counted as freezes: Cra was in a loop of its own on purpose.\(longest)"))
+ }
+ if r.selfCost.cpuPct > selfCostBudgetPct {
+ out.append(PerfInsight(id: "profiler-overhead", severity: .info,
+ title: "Recording costs \(String(format: "%.3f", r.selfCost.cpuPct)) % of a core",
+ detail: "Over its \(selfCostBudgetPct) % budget. The numbers still hold; selfCost in perf.json says which part costs it."))
+ }
+ if r.idle.samples >= idleMinSamples, r.idle.cpuPct >= idleCpuPct {
+ out.append(PerfInsight(id: "idle-cpu", severity: .info,
+ title: "Cra kept working while the Mac was idle",
+ detail: "\(Int(r.idle.cpuPct.rounded())) % of a core on average over \(r.idle.samples * 5 / 60) min without keyboard or mouse input. A playing video counts too."))
}
let crashes = r.counters[PerfCounter.rendererTerminated.rawValue]?.values.reduce(0, +) ?? 0
if crashes > 0 {
@@ -96,8 +118,13 @@ public enum PerfSummary {
let m = r.mainThread
var lines = [
"Cra \(r.session.craVersion), Chromium \(r.session.chromiumVersion), launch \(r.session.launch), up \(Int(r.session.uptimeSeconds / 60)) min",
- "Main thread: \(m.stalls) stalls ≥250 ms, \(m.longStalls) ≥1 s, \(m.hangs) ≥2 s, longest \(PerfInsights.duration(m.longestMs)) (p99 pass \(PerfInsights.duration(m.passes.percentile(0.99))))",
+ "Main thread: \(m.freezes) freezes ≥250 ms (\(m.longFreezes) ≥1 s, \(m.hangs) ≥2 s), longest freeze \(PerfInsights.duration(m.longestFreezeMs)); "
+ + "longest pass \(PerfInsights.duration(m.longestPassMs))\(m.longestPassMode.map { " (\($0))" } ?? ""), "
+ + "\(m.outsideFreezes) outside freezes (menus, drags, alerts, start-up); p99 pass \(PerfInsights.duration(m.passes.percentile(0.99)))",
]
+ if let s = m.slowPass {
+ lines.append("Slow pass sampled: \(PerfInsights.duration(s.endedMs ?? s.stalledMs)) (\(s.mode)), top: \(s.stack.first?.trimmingCharacters(in: .whitespaces) ?? s.stackNote ?? "no stack")")
+ }
if let p = r.session.previous { lines.append("Previous session: \(p.abnormal ? "ended without quitting" : "quit normally") (Cra \(p.craVersion))") }
for g in r.processes where g.count > 0 {
lines.append("\(g.kind.rawValue) ×\(g.count): \(String(format: "%.1f", g.cpuPct)) % CPU, \(PerfInsights.bytes(g.footprintBytes))")
@@ -114,6 +141,10 @@ public enum PerfSummary {
}
}
if r.tabs.longTasks > 0 { lines.append("Page long tasks: \(r.tabs.longTasks), \(PerfInsights.duration(r.tabs.longTaskMs)) in all") }
+ if r.tabs.longFrames > 0 { lines.append("Page long frames: \(r.tabs.longFrames), longest \(PerfInsights.duration(r.tabs.longestFrameMs))") }
+ if r.tabs.videoFrames > 0 { lines.append("Video: \(r.tabs.droppedVideoFrames) of \(r.tabs.videoFrames) frames dropped") }
+ if r.idle.samples > 0 { lines.append("Idle: \(String(format: "%.1f", r.idle.cpuPct)) % CPU over \(r.idle.samples * 5 / 60) min") }
+ lines.append("Recording: \(String(format: "%.3f", r.selfCost.cpuPct)) % of a core")
for (counter, keys) in r.counters.sorted(by: { $0.key < $1.key }) where !keys.isEmpty {
lines.append("\(counter): " + keys.sorted { $0.key < $1.key }.map { "\($0.key) ×\($0.value)" }.joined(separator: ", "))
}
diff --git a/Packages/CraCore/Sources/CraCore/Perf/PerfMetric.swift b/Packages/CraCore/Sources/CraCore/Perf/PerfMetric.swift
index 2d6a431..81351fa 100644
--- a/Packages/CraCore/Sources/CraCore/Perf/PerfMetric.swift
+++ b/Packages/CraCore/Sources/CraCore/Perf/PerfMetric.swift
@@ -3,9 +3,6 @@ import Foundation
/// What the performance recorder times, in milliseconds (GitHub #91). The raw values are the
/// keys in perf.json, so renaming one changes the file's schema.
public enum PerfMetric: String, Codable, CaseIterable, Sendable {
- /// One pass of the main run loop between two points where it could take input. The main
- /// thread is also Chromium's browser UI thread: a long pass is a frozen window.
- case mainLoop
/// Putting other tabs on screen (BrowserManager.show), the part on the main thread.
case tabSwitch
/// Making a tab's page: Chromium's view, its window and the browser object.
@@ -14,7 +11,8 @@ public enum PerfMetric: String, Codable, CaseIterable, Sendable {
case pageLoad
/// A tab that tab sleeping closed, shown again until its page finished loading.
case tabWake
- /// A data-folder JSON file decoded at launch, or encoded and written.
+ /// A data-folder JSON file decoded at launch, or encoded and written. Saves run on the
+ /// persistence queue, so `jsonSave` is that queue's time, never a main-thread stall.
case jsonLoad
case jsonSave
/// Long tasks (50 ms or more) in the visible tab's page.
diff --git a/Packages/CraCore/Sources/CraCore/Perf/PerfReport.swift b/Packages/CraCore/Sources/CraCore/Perf/PerfReport.swift
index e9110e5..ecfab5f 100644
--- a/Packages/CraCore/Sources/CraCore/Perf/PerfReport.swift
+++ b/Packages/CraCore/Sources/CraCore/Perf/PerfReport.swift
@@ -5,8 +5,10 @@ import Foundation
/// appear where something happened on one (a stall while it was on screen, a renderer that
/// crashed) and become `site-N` with "Hide site names".
public struct PerfReport: Codable, Equatable, Sendable {
- /// Bumped when a field changes meaning or goes away; new fields don't bump it.
- public static let currentSchema = 1
+ /// Bumped when a field changes meaning or goes away; new fields don't bump it. 2: a pass ends
+ /// only where Core Animation commits, the freeze counts say so in their names, one clock
+ /// for `at` and `startedAt` (GitHub #135).
+ public static let currentSchema = 2
public var schema = PerfReport.currentSchema
public var generatedAt: Date
@@ -21,40 +23,78 @@ public struct PerfReport: Codable, Equatable, Sendable {
public var startup: [String: Double]
public var storage: Storage
public var events: [PerfEvent]
+ public var idle: Idle
public var insights: [PerfInsight]
+ /// Every pass is in `passes`. A stall (250 ms or more) in the default mode is a freeze; one in
+ /// a menu, a drag, an alert or start-up is counted in `outsideFreezes` instead, and both are
+ /// in `longestPassMs`, the honest answer to how long the main thread was blocked.
public struct MainThread: Codable, Equatable, Sendable {
public var passes: LogHistogram
- public var stalls: Int
- public var longStalls: Int
+ public var freezes: Int
+ /// Freezes of 1 s or more, and of 2 s or more.
+ public var longFreezes: Int
public var hangs: Int
- /// Stalls in a tracking or modal loop (a held menu, a drag, an alert): not freezes.
public var outsideFreezes: Int
- public var longestMs: Double
+ public var longestFreezeMs: Double
+ public var longestPassMs: Double
+ /// The mode of the longest pass that was a stall (nil: none was).
+ public var longestPassMode: String?
public var recent: [PerfStall]
+ /// The session's first pass that lasted `slowStackMs` while the watchdog looked, with the
+ /// main thread's stack from `sample` (number 0: not a hang file).
+ public var slowPass: PerfHang?
- public init(passes: LogHistogram = LogHistogram(), stalls: Int = 0, longStalls: Int = 0, hangs: Int = 0,
- outsideFreezes: Int = 0, longestMs: Double = 0, recent: [PerfStall] = []) {
+ public init(passes: LogHistogram = LogHistogram(), freezes: Int = 0, longFreezes: Int = 0, hangs: Int = 0,
+ outsideFreezes: Int = 0, longestFreezeMs: Double = 0, longestPassMs: Double = 0, longestPassMode: String? = nil,
+ recent: [PerfStall] = [], slowPass: PerfHang? = nil) {
self.passes = passes
- self.stalls = stalls
- self.longStalls = longStalls
+ self.freezes = freezes
+ self.longFreezes = longFreezes
self.hangs = hangs
self.outsideFreezes = outsideFreezes
- self.longestMs = longestMs
+ self.longestFreezeMs = longestFreezeMs
+ self.longestPassMs = longestPassMs
+ self.longestPassMode = longestPassMode
self.recent = recent
+ self.slowPass = slowPass
}
}
+ /// The visible tab's page, summed since launch (history.json has long tasks per 5 minutes).
public struct Tabs: Codable, Equatable, Sendable {
public var latest: PerfTabCounts?
- /// The page long tasks in the visible tab, summed since launch.
public var longTasks: Int
public var longTaskMs: Double
+ /// Chromium's long animation frames (50 ms or more from input or rAF to the frame), and
+ /// the longest: frames the page was late with.
+ public var longFrames: Int
+ public var longestFrameMs: Double
+ /// `getVideoPlaybackQuality()` over the page's videos while they played on screen.
+ public var videoFrames: Int
+ public var droppedVideoFrames: Int
- public init(latest: PerfTabCounts? = nil, longTasks: Int = 0, longTaskMs: Double = 0) {
+ public init(latest: PerfTabCounts? = nil, longTasks: Int = 0, longTaskMs: Double = 0, longFrames: Int = 0,
+ longestFrameMs: Double = 0, videoFrames: Int = 0, droppedVideoFrames: Int = 0) {
self.latest = latest
self.longTasks = longTasks
self.longTaskMs = longTaskMs
+ self.longFrames = longFrames
+ self.longestFrameMs = longestFrameMs
+ self.videoFrames = videoFrames
+ self.droppedVideoFrames = droppedVideoFrames
+ }
+ }
+
+ /// Cra and its helpers while nobody used the Mac (no keyboard or mouse input for a minute).
+ public struct Idle: Codable, Equatable, Sendable {
+ /// 5-second samples taken while idle, and Cra's average CPU over them (percent of a core).
+ public var samples: Int
+ public var cpuPct: Double
+
+ public init(samples: Int = 0, cpuPct: Double = 0) {
+ self.samples = samples
+ self.cpuPct = cpuPct
}
}
@@ -74,7 +114,7 @@ public struct PerfReport: Codable, Equatable, Sendable {
public init(generatedAt: Date, session: PerfSessionInfo, selfCost: PerfSelfCost, mainThread: MainThread,
processes: [PerfProcessGroup], system: PerfSystemSample?, tabs: Tabs, timings: [String: LogHistogram],
counters: [String: [String: Int]], startup: [String: Double], storage: Storage, events: [PerfEvent],
- insights: [PerfInsight]) {
+ idle: Idle = Idle(), insights: [PerfInsight]) {
self.generatedAt = generatedAt
self.session = session
self.selfCost = selfCost
@@ -87,6 +127,7 @@ public struct PerfReport: Codable, Equatable, Sendable {
self.startup = startup
self.storage = storage
self.events = events
+ self.idle = idle
self.insights = insights
}
@@ -108,6 +149,7 @@ public struct PerfReport: Codable, Equatable, Sendable {
public var sites: Set<String> {
var s = Set(mainThread.recent.compactMap(\.site))
s.formUnion(events.compactMap(\.site))
+ if let site = mainThread.slowPass?.site { s.insert(site) }
return s
}
}
@@ -118,8 +160,12 @@ public struct PerfSessionInfo: Codable, Equatable, Sendable {
public var id: String
/// How many times this data folder has been started, this launch included.
public var launch: Int
+ /// When Cra started (the top of main.swift), to the second. Every `at` in the report is seconds
+ /// since then on a clock that counts the Mac's sleep, so `startedAt + at` is the time in cra.log.
public var startedAt: Date
+ /// Since `startedAt`, and how much of it the Mac slept.
public var uptimeSeconds: Double
+ public var asleepSeconds: Double
public var craVersion: String
public var chromiumVersion: String
/// This launch ran the onboarding (a fresh install, or one that never finished it).
@@ -129,12 +175,13 @@ public struct PerfSessionInfo: Codable, Equatable, Sendable {
/// Whether this Cra writes files under Perf/ (it owns the data folder and recording is on).
public var writesFiles: Bool
- public init(id: String, launch: Int, startedAt: Date, uptimeSeconds: Double, craVersion: String, chromiumVersion: String,
- onboarding: Bool, firstLaunchAt: Date?, previous: PerfPreviousSession?, writesFiles: Bool) {
+ public init(id: String, launch: Int, startedAt: Date, uptimeSeconds: Double, asleepSeconds: Double = 0, craVersion: String,
+ chromiumVersion: String, onboarding: Bool, firstLaunchAt: Date?, previous: PerfPreviousSession?, writesFiles: Bool) {
self.id = id
self.launch = launch
self.startedAt = startedAt
self.uptimeSeconds = uptimeSeconds
+ self.asleepSeconds = asleepSeconds
self.craVersion = craVersion
self.chromiumVersion = chromiumVersion
self.onboarding = onboarding
@@ -142,26 +189,43 @@ public struct PerfSessionInfo: Codable, Equatable, Sendable {
self.previous = previous
self.writesFiles = writesFiles
}
+
+ /// Awake time: what rates per hour and CPU shares are taken over.
+ public var awakeSeconds: Double { max(uptimeSeconds - asleepSeconds, 0) }
}
-/// What the profiler itself costs, so CI can hold it to its budget (under 0.1 % of a core).
+/// What the profiler itself costs, so CI can hold it to its budget (`PerfInsights.selfCostBudgetPct`).
public struct PerfSelfCost: Codable, Equatable, Sendable {
+ /// All of it, as a share of one core over the awake time recording was on.
public var cpuMs: Double
public var cpuPct: Double
- /// Main run-loop callbacks observed, and their estimated cost (included in `cpuMs`).
+ /// Main run-loop callbacks observed, and their estimated cost.
public var observerCallbacks: Int
public var observerMs: Double
+ /// The rest of `cpuMs` by part: the watchdog, the 5-second process and Mac samples, the black
+ /// box and history (building, encoding, writing) and the main thread's tab count and page read.
+ public var watchdogMs: Double
+ public var processSampleMs: Double
+ public var systemSampleMs: Double
+ public var filesMs: Double
+ public var mainSamplerMs: Double
public var bytesWritten: Int
public var writes: Int
/// The recorder's own data in memory, estimated from its counts.
public var memoryBytes: Int
- public init(cpuMs: Double = 0, cpuPct: Double = 0, observerCallbacks: Int = 0, observerMs: Double = 0, bytesWritten: Int = 0,
- writes: Int = 0, memoryBytes: Int = 0) {
+ public init(cpuMs: Double = 0, cpuPct: Double = 0, observerCallbacks: Int = 0, observerMs: Double = 0, watchdogMs: Double = 0,
+ processSampleMs: Double = 0, systemSampleMs: Double = 0, filesMs: Double = 0, mainSamplerMs: Double = 0,
+ bytesWritten: Int = 0, writes: Int = 0, memoryBytes: Int = 0) {
self.cpuMs = cpuMs
self.cpuPct = cpuPct
self.observerCallbacks = observerCallbacks
self.observerMs = observerMs
+ self.watchdogMs = watchdogMs
+ self.processSampleMs = processSampleMs
+ self.systemSampleMs = systemSampleMs
+ self.filesMs = filesMs
+ self.mainSamplerMs = mainSamplerMs
self.bytesWritten = bytesWritten
self.writes = writes
self.memoryBytes = memoryBytes
diff --git a/Packages/CraCore/Sources/CraCore/Perf/PerfStall.swift b/Packages/CraCore/Sources/CraCore/Perf/PerfStall.swift
index 7225854..628a968 100644
--- a/Packages/CraCore/Sources/CraCore/Perf/PerfStall.swift
+++ b/Packages/CraCore/Sources/CraCore/Perf/PerfStall.swift
@@ -26,8 +26,13 @@ public struct PerfStall: Codable, Equatable, Sendable {
public var marks: [String]
/// The host of the page on screen, nil for incognito or none.
public var site: String?
+ /// Whether Cra was the active app, and whether any of its windows showed on screen (not
+ /// covered, minimised or on another Space), when the stall ended; nil before Cra said.
+ public var active: Bool?
+ public var onScreen: Bool?
- public init(at: Double, ms: Double, kind: Kind, mode: String, freeze: Bool, marks: [String], site: String?) {
+ public init(at: Double, ms: Double, kind: Kind, mode: String, freeze: Bool, marks: [String], site: String?,
+ active: Bool? = nil, onScreen: Bool? = nil) {
self.at = at
self.ms = ms
self.kind = kind
@@ -35,6 +40,8 @@ public struct PerfStall: Codable, Equatable, Sendable {
self.freeze = freeze
self.marks = marks
self.site = site
+ self.active = active
+ self.onScreen = onScreen
}
}
@@ -43,6 +50,9 @@ public struct PerfStallPolicy: Equatable, Sendable {
public var stallMs: Double = 250
public var longMs: Double = 1000
public var hangMs: Double = 2000
+ /// A pass this long, still going, gets the session's one early stack: no real session in the
+ /// first exports reached 2 s, so the hang stacks never ran (GitHub #135).
+ public var slowStackMs: Double = 1000
/// Marks this close before the end of a stall are its breadcrumbs.
public var breadcrumbWindow: Double = 5
public var breadcrumbLimit = 8
@@ -72,12 +82,18 @@ public struct PerfStallPolicy: Equatable, Sendable {
public static func isFreeze(mode: String) -> Bool { !["tracking", "modal", "launch"].contains(mode) }
/// The stall a pass of `ms` ending at `at` makes, with the marks of the seconds before it.
- public func stall(at: Double, ms: Double, mode: String, marks: [PerfMark], site: String?) -> PerfStall? {
+ public func stall(at: Double, ms: Double, mode: String, marks: [PerfMark], site: String?,
+ active: Bool? = nil, onScreen: Bool? = nil) -> PerfStall? {
guard let kind = kind(ms: ms) else { return nil }
let short = Self.shortMode(mode)
+ return PerfStall(at: at, ms: ms, kind: kind, mode: short, freeze: Self.isFreeze(mode: short),
+ marks: breadcrumbs(at: at, ms: ms, marks: marks), site: site, active: active, onScreen: onScreen)
+ }
+
+ /// The marks of the seconds before a pass of `ms` that ended (or was seen) at `at`.
+ public func breadcrumbs(at: Double, ms: Double, marks: [PerfMark]) -> [String] {
let since = at - ms / 1000 - breadcrumbWindow
- let crumbs = marks.filter { $0.at >= since && $0.at <= at }.suffix(breadcrumbLimit).map(\.label)
- return PerfStall(at: at, ms: ms, kind: kind, mode: short, freeze: Self.isFreeze(mode: short), marks: Array(crumbs), site: site)
+ return Array(marks.filter { $0.at >= since && $0.at <= at }.suffix(breadcrumbLimit).map(\.label))
}
}
diff --git a/Packages/CraCore/Sources/CraCore/Perf/PerfState.swift b/Packages/CraCore/Sources/CraCore/Perf/PerfState.swift
index 7d77652..5014b25 100644
--- a/Packages/CraCore/Sources/CraCore/Perf/PerfState.swift
+++ b/Packages/CraCore/Sources/CraCore/Perf/PerfState.swift
@@ -2,28 +2,38 @@ import Foundation
/// Everything the recorder holds in memory for one session, and the report made from it. The
/// recorder keeps one behind a lock; every change is a call here, so what a report says is
-/// decided (and tested) in one place. Times are seconds since the session started.
+/// decided (and tested) in one place. Times are seconds since Cra started (PerfSessionInfo.startedAt).
public struct PerfState: Sendable {
public var policy = PerfStallPolicy()
public private(set) var passes = LogHistogram()
public private(set) var timings: [PerfMetric: LogHistogram] = [:]
public private(set) var counters: [PerfCounter: [String: Int]] = [:]
public private(set) var stalls = PerfRing<PerfStall>(capacity: 40)
- public private(set) var stallCount = 0
- public private(set) var longStallCount = 0
+ public private(set) var freezeCount = 0
+ public private(set) var longFreezeCount = 0
public private(set) var hangCount = 0
public private(set) var outsideFreezes = 0
- public private(set) var longestMs: Double = 0
+ public private(set) var longestFreezeMs: Double = 0
+ /// The longest stall of any mode, and its mode.
+ public private(set) var longestStall: (ms: Double, mode: String)?
+ public var slowPass: PerfHang?
public private(set) var marks = PerfRing<PerfMark>(capacity: 64)
public private(set) var events = PerfRing<PerfEvent>(capacity: 150)
public private(set) var longTasks = 0
public private(set) var longTaskMs: Double = 0
+ public private(set) var page = PerfPageFrames()
+ public private(set) var idle = PerfReport.Idle()
public private(set) var startup: [String: Double] = [:]
public var processes: [PerfProcessGroup] = []
public var system: PerfSystemSample?
public var tabs: PerfTabCounts?
/// The host of the page Cra put on screen last (nil for none or incognito), for stalls.
public var visibleSite: String?
+ /// Cra the active app, and a window of it on screen; nil until the app says.
+ public var appActive: Bool?
+ public var appOnScreen: Bool?
+ /// No keyboard or mouse input on the Mac for a minute, as of the last main-thread sample.
+ public var userIdle = false
private var historyBase = (stalls: 0, long: 0, longTasks: 0, crashes: 0)
public init() {}
@@ -39,13 +49,15 @@ public struct PerfState: Sendable {
/// A pass of `ms` that ended at `at`: counted, and returned when it is a stall.
@discardableResult
public mutating func pass(ms: Double, endedAt at: Double, mode: String) -> PerfStall? {
- guard let s = policy.stall(at: at, ms: ms, mode: mode, marks: marks.elements, site: visibleSite) else { return nil }
+ guard let s = policy.stall(at: at, ms: ms, mode: mode, marks: marks.elements, site: visibleSite,
+ active: appActive, onScreen: appOnScreen) else { return nil }
stalls.append(s)
+ if ms > longestStall?.ms ?? 0 { longestStall = (ms, s.mode) }
guard s.freeze else { outsideFreezes += 1; return s }
- stallCount += 1
- if s.kind != .stall { longStallCount += 1 }
+ freezeCount += 1
+ if s.kind != .stall { longFreezeCount += 1 }
if s.kind == .hang { hangCount += 1 }
- longestMs = max(longestMs, ms)
+ longestFreezeMs = max(longestFreezeMs, ms)
if s.kind != .stall {
events.append(PerfEvent(at: at, kind: s.kind == .hang ? "hang" : "freeze",
detail: s.marks.last.map { "after \($0)" } ?? "", ms: ms, site: s.site))
@@ -73,6 +85,16 @@ public struct PerfState: Sendable {
for d in durations.prefix(100) { time(.longTask, ms: d) }
}
+ /// The visible tab's late frames and video frames since the last read.
+ public mutating func pageFrames(_ f: PerfPageFrames) { page.add(f) }
+
+ /// A 5-second sample of Cra's CPU (percent of a core, all processes), counted when idle.
+ public mutating func idleSample(cpuPct: Double) {
+ guard userIdle else { return }
+ idle.cpuPct = (idle.cpuPct * Double(idle.samples) + cpuPct) / Double(idle.samples + 1)
+ idle.samples += 1
+ }
+
/// A start-up phase, the first time it is reached (milliseconds since the process started).
public mutating func phase(_ name: String, ms: Double) {
if startup[name] == nil { startup[name] = ms }
@@ -83,10 +105,10 @@ public struct PerfState: Sendable {
let crashes = counters[.rendererTerminated]?.values.reduce(0, +) ?? 0
let s = PerfHistorySample(cpuPct: processes.reduce(0) { $0 + $1.cpuPct },
footprintBytes: processes.reduce(0) { $0 + $1.footprintBytes },
- stalls: stallCount - historyBase.stalls, longStalls: longStallCount - historyBase.long,
+ stalls: freezeCount - historyBase.stalls, longStalls: longFreezeCount - historyBase.long,
longTasks: longTasks - historyBase.longTasks, rendererCrashes: crashes - historyBase.crashes,
liveTabs: tabs?.live ?? 0)
- historyBase = (stallCount, longStallCount, longTasks, crashes)
+ historyBase = (freezeCount, longFreezeCount, longTasks, crashes)
return s
}
@@ -96,13 +118,17 @@ public struct PerfState: Sendable {
if storage.freeDiskBytes == nil { storage.freeDiskBytes = system?.freeDiskBytes }
var r = PerfReport(
generatedAt: generatedAt, session: session, selfCost: selfCost,
- mainThread: .init(passes: passes, stalls: stallCount, longStalls: longStallCount, hangs: hangCount,
- outsideFreezes: outsideFreezes, longestMs: longestMs, recent: stalls.elements),
+ mainThread: .init(passes: passes, freezes: freezeCount, longFreezes: longFreezeCount, hangs: hangCount,
+ outsideFreezes: outsideFreezes, longestFreezeMs: longestFreezeMs,
+ // The histogram is pulled every 5 s; a stall is known as soon as it ends.
+ longestPassMs: max(passes.maxMs, longestStall?.ms ?? 0), longestPassMode: longestStall?.mode,
+ recent: stalls.elements, slowPass: slowPass),
processes: processes, system: system,
- tabs: .init(latest: tabs, longTasks: longTasks, longTaskMs: longTaskMs),
+ tabs: .init(latest: tabs, longTasks: longTasks, longTaskMs: longTaskMs, longFrames: page.longFrames,
+ longestFrameMs: page.longestFrameMs, videoFrames: page.videoFrames, droppedVideoFrames: page.droppedVideoFrames),
timings: Dictionary(uniqueKeysWithValues: timings.map { ($0.key.rawValue, $0.value) }),
counters: Dictionary(uniqueKeysWithValues: counters.map { ($0.key.rawValue, $0.value) }),
- startup: startup, storage: storage, events: events.elements, insights: [])
+ startup: startup, storage: storage, events: events.elements, idle: idle, insights: [])
r.insights = PerfInsights.evaluate(r)
return r
}
@@ -113,6 +139,32 @@ public struct PerfState: Sendable {
let marksBytes = marks.elements.reduce(0) { $0 + 32 + $1.label.utf8.count }
let stallBytes = stalls.elements.reduce(0) { $0 + 96 + $1.marks.reduce(0) { $0 + $1.utf8.count + 16 } }
let eventBytes = events.elements.reduce(0) { $0 + 80 + $1.detail.utf8.count + ($1.site?.utf8.count ?? 0) }
- return 1024 + histograms + marksBytes + stallBytes + eventBytes + processes.count * 96
+ let slowBytes = slowPass.map { 128 + $0.stack.reduce(0) { $0 + $1.utf8.count + 16 } } ?? 0
+ return 1024 + histograms + marksBytes + stallBytes + eventBytes + slowBytes + processes.count * 96
+ }
+}
+
+/// What the visible page reports about its frames between two reads (PageScript's `__cra.lf`
+/// and `getVideoPlaybackQuality()`), added up over the session.
+public struct PerfPageFrames: Equatable, Sendable {
+ public var longFrames = 0
+ public var longestFrameMs: Double = 0
+ public var videoFrames = 0
+ public var droppedVideoFrames = 0
+
+ public init(longFrames: Int = 0, longestFrameMs: Double = 0, videoFrames: Int = 0, droppedVideoFrames: Int = 0) {
+ self.longFrames = longFrames
+ self.longestFrameMs = longestFrameMs
+ self.videoFrames = videoFrames
+ self.droppedVideoFrames = droppedVideoFrames
+ }
+
+ /// A page can answer garbage: negative counts are dropped, a dropped count never exceeds the frames.
+ public mutating func add(_ f: PerfPageFrames) {
+ longFrames += max(0, f.longFrames)
+ if f.longestFrameMs.isFinite { longestFrameMs = max(longestFrameMs, f.longestFrameMs) }
+ let total = max(0, f.videoFrames)
+ videoFrames += total
+ droppedVideoFrames += min(max(0, f.droppedVideoFrames), total)
}
}
diff --git a/Packages/CraCore/Tests/CraCoreTests/Fixtures/Perf/perf-golden.json b/Packages/CraCore/Tests/CraCoreTests/Fixtures/Perf/perf-golden.json
index 1ca55b9..3f804ef 100644
--- a/Packages/CraCore/Tests/CraCoreTests/Fixtures/Perf/perf-golden.json
+++ b/Packages/CraCore/Tests/CraCoreTests/Fixtures/Perf/perf-golden.json
@@ -31,6 +31,10 @@
}
],
"generatedAt" : "2026-10-26T07:43:20Z",
+ "idle" : {
+ "cpuPct" : 3,
+ "samples" : 1
+ },
"insights" : [
{
"detail" : "The longest lasted 1.2 s.",
@@ -38,6 +42,12 @@
"severity" : "warning",
"title" : "Cra froze 1 time for a second or more"
},
+ {
+ "detail" : "Not counted as freezes: Cra was in a loop of its own on purpose. The longest lasted 4.0 s (tracking).",
+ "id" : "outside-freezes",
+ "severity" : "info",
+ "title" : "1 long pass in menus, drags, alerts or start-up"
+ },
{
"detail" : "A page's process crashed, ran out of memory or was ended.",
"id" : "renderer-crashes",
@@ -46,9 +56,12 @@
}
],
"mainThread" : {
+ "freezes" : 2,
"hangs" : 0,
- "longStalls" : 1,
- "longestMs" : 1200,
+ "longFreezes" : 1,
+ "longestFreezeMs" : 1200,
+ "longestPassMode" : "tracking",
+ "longestPassMs" : 4000,
"outsideFreezes" : 1,
"passes" : {
"buckets" : [
@@ -87,6 +100,7 @@
"site" : "video.example.com"
},
{
+ "active" : true,
"at" : 101.7,
"freeze" : true,
"kind" : "long",
@@ -96,9 +110,11 @@
],
"mode" : "default",
"ms" : 1200,
+ "onScreen" : true,
"site" : "video.example.com"
},
{
+ "active" : true,
"at" : 200,
"freeze" : false,
"kind" : "hang",
@@ -107,10 +123,27 @@
],
"mode" : "tracking",
"ms" : 4000,
+ "onScreen" : true,
"site" : "video.example.com"
}
],
- "stalls" : 2
+ "slowPass" : {
+ "at" : 100.5,
+ "endedMs" : 1200,
+ "marks" : [
+ "swipe-begin",
+ "page-shown"
+ ],
+ "mode" : "default",
+ "number" : 0,
+ "session" : "0123abcd",
+ "site" : "video.example.com",
+ "stack" : [
+ "10 main [Cra]",
+ " 9 layout() [Cra]"
+ ],
+ "stalledMs" : 1000
+ }
},
"processes" : [
{
@@ -138,17 +171,23 @@
"wakeups" : 5000
}
],
- "schema" : 1,
+ "schema" : 2,
"selfCost" : {
"bytesWritten" : 70000,
"cpuMs" : 120,
"cpuPct" : 0.02,
+ "filesMs" : 0,
+ "mainSamplerMs" : 0,
"memoryBytes" : 9000,
"observerCallbacks" : 50000,
"observerMs" : 4,
+ "processSampleMs" : 0,
+ "systemSampleMs" : 0,
+ "watchdogMs" : 0,
"writes" : 2
},
"session" : {
+ "asleepSeconds" : 0,
"chromiumVersion" : "156.0.7445.0",
"craVersion" : "0.1.27",
"firstLaunchAt" : "2026-10-25T07:33:20Z",
@@ -176,6 +215,7 @@
"thermalState" : "nominal"
},
"tabs" : {
+ "droppedVideoFrames" : 12,
"latest" : {
"incognito" : 0,
"live" : 6,
@@ -184,8 +224,11 @@
"visible" : 1,
"windows" : 1
},
+ "longFrames" : 4,
"longTaskMs" : 420,
- "longTasks" : 3
+ "longTasks" : 3,
+ "longestFrameMs" : 180,
+ "videoFrames" : 600
},
"timings" : {
"longTask" : {
diff --git a/Packages/CraCore/Tests/CraCoreTests/PerfStateTests.swift b/Packages/CraCore/Tests/CraCoreTests/PerfStateTests.swift
index 17bce78..cd16ab3 100644
--- a/Packages/CraCore/Tests/CraCoreTests/PerfStateTests.swift
+++ b/Packages/CraCore/Tests/CraCoreTests/PerfStateTests.swift
@@ -22,6 +22,8 @@ struct PerfStateTests {
s.mark("swipe-begin", at: 100)
s.mark("page-shown", at: 100.5)
s.pass(ms: 300, endedAt: 50, mode: "kCFRunLoopDefaultMode")
+ s.appActive = true
+ s.appOnScreen = true
s.pass(ms: 1200, endedAt: 101.7, mode: "kCFRunLoopDefaultMode")
s.pass(ms: 4000, endedAt: 200, mode: "NSEventTrackingRunLoopMode")
s.time(.tabSwitch, ms: 6)
@@ -31,6 +33,13 @@ struct PerfStateTests {
s.count(.loadError, key: "-105", at: 310)
s.count(.sleepRefusal, key: "typed", at: 320)
s.pageLongTasks(count: 3, totalMs: 420, durations: [60, 120, 240])
+ s.pageFrames(PerfPageFrames(longFrames: 4, longestFrameMs: 180, videoFrames: 600, droppedVideoFrames: 12))
+ s.slowPass = PerfHang(session: "0123abcd", number: 0, at: 100.5, stalledMs: 1000, endedMs: 1200, mode: "default",
+ marks: ["swipe-begin", "page-shown"], site: "video.example.com", stack: ["10 main [Cra]", " 9 layout() [Cra]"])
+ s.userIdle = true
+ s.idleSample(cpuPct: 3)
+ s.userIdle = false
+ s.idleSample(cpuPct: 50)
s.phase("main", ms: 12)
s.phase("cefStarted", ms: 480)
s.phase("main", ms: 999)
@@ -53,14 +62,19 @@ struct PerfStateTests {
@Test func stallsAreCountedByKindAndHeldMenusAreNotFreezes() {
let r = Self.sampleReport()
let m = r.mainThread
- #expect(m.stalls == 2)
- #expect(m.longStalls == 1)
+ #expect(m.freezes == 2)
+ #expect(m.longFreezes == 1)
#expect(m.hangs == 0)
#expect(m.outsideFreezes == 1)
- #expect(m.longestMs == 1200)
+ #expect(m.longestFreezeMs == 1200)
+ #expect(m.longestPassMs == 4000 && m.longestPassMode == "tracking")
#expect(m.recent.count == 3)
#expect(m.recent[1].marks == ["swipe-begin", "page-shown"])
#expect(m.recent[1].site == "video.example.com")
+ #expect(m.recent[0].active == nil && m.recent[1].active == true && m.recent[1].onScreen == true)
+ #expect(m.slowPass?.endedMs == 1200)
+ #expect(r.tabs.longFrames == 4 && r.tabs.droppedVideoFrames == 12)
+ #expect(r.idle == PerfReport.Idle(samples: 1, cpuPct: 3))
#expect(m.passes.count == 6)
#expect(r.events.map(\.kind) == ["freeze", "renderer", "load"])
#expect(r.events[0].detail == "after page-shown")
@@ -77,7 +91,7 @@ struct PerfStateTests {
#expect(s.pass(ms: 100, endedAt: 1, mode: "kCFRunLoopDefaultMode") == nil)
let hang = s.pass(ms: 2500, endedAt: 3, mode: "kCFRunLoopDefaultMode")
#expect(hang?.kind == .hang)
- #expect(s.stallCount == 1 && s.longStallCount == 1 && s.hangCount == 1)
+ #expect(s.freezeCount == 1 && s.longFreezeCount == 1 && s.hangCount == 1)
#expect(s.events.elements.map(\.kind) == ["hang"])
s.count(.rendererUnresponsive, key: "", at: 4)
s.count(.rendererResponsive, key: "", at: 5)
@@ -148,29 +162,53 @@ struct PerfInsightTests {
let r = report {
$0.session.previous = PerfPreviousSession(id: "x", launch: 1, startedAt: PerfStateTests.start, craVersion: "0.1.26",
abnormal: true, endedAt: nil)
- $0.mainThread.longStalls = 2
+ $0.mainThread.longFreezes = 2
$0.mainThread.hangs = 1
- $0.mainThread.longestMs = 2600
- $0.mainThread.stalls = 40
+ $0.mainThread.longestFreezeMs = 2600
+ $0.mainThread.freezes = 40
+ $0.mainThread.outsideFreezes = 3
+ $0.mainThread.longestPassMs = 3100
+ $0.mainThread.longestPassMode = "modal"
+ $0.selfCost.cpuPct = 0.145
+ $0.idle = PerfReport.Idle(samples: 24, cpuPct: 31)
$0.counters = ["rendererTerminated": ["crashed": 1], "rendererUnresponsive": ["": 2]]
$0.system = PerfSystemSample(cpuPct: 90, memoryPressure: .critical, thermalState: "serious", lowPowerMode: true,
freeDiskBytes: 1 << 30)
$0.processes = [PerfProcessGroup(kind: .renderer, count: 30, footprintBytes: 9 << 30)]
$0.tabs.latest = PerfTabCounts(tabs: 100, live: 30, visible: 1, slept: 0, incognito: 0, windows: 2)
}
- #expect(r.insights.map(\.id) == ["previous-abnormal", "freezes", "stalls", "renderer-crashes", "unresponsive",
- "memory-pressure", "thermal", "low-power", "low-disk", "footprint"])
+ #expect(r.insights.map(\.id) == ["previous-abnormal", "freezes", "stalls", "outside-freezes", "profiler-overhead", "idle-cpu",
+ "renderer-crashes", "unresponsive", "memory-pressure", "thermal", "low-power", "low-disk", "footprint"])
#expect(r.insights[1].title == "Cra froze 2 times for a second or more")
#expect(r.insights[1].detail == "The longest lasted 2.6 s; 1 went past 2 s.")
- #expect(r.insights[3].title == "1 page stopped (“Aw, Snap!”)")
- #expect(r.insights[8].detail.hasPrefix("1.0 GB free"))
- #expect(r.insights[9].title == "Cra uses 9.0 GB of memory")
- #expect(r.insights[9].detail.contains("Tab sleeping (Settings ▸ Tabs)"))
+ #expect(r.insights[3].title == "3 long passes in menus, drags, alerts or start-up")
+ #expect(r.insights[3].detail.hasSuffix("The longest lasted 3.1 s (modal)."))
+ #expect(r.insights[4].title == "Recording costs 0.145 % of a core")
+ #expect(r.insights[5].detail.hasPrefix("31 % of a core on average over 2 min"))
+ #expect(r.insights[6].title == "1 page stopped (“Aw, Snap!”)")
+ #expect(r.insights[11].detail.hasPrefix("1.0 GB free"))
+ #expect(r.insights[12].title == "Cra uses 9.0 GB of memory")
+ #expect(r.insights[12].detail.contains("Tab sleeping (Settings ▸ Tabs)"))
}
@Test func aFewStallsInALongSessionAreNotFrequent() {
- #expect(report { $0.mainThread.stalls = 5; $0.session.uptimeSeconds = 3600 }.insights.isEmpty)
- #expect(report { $0.mainThread.stalls = 5; $0.session.uptimeSeconds = 300 }.insights.map(\.id) == ["stalls"])
+ #expect(report { $0.mainThread.freezes = 5; $0.session.uptimeSeconds = 3600 }.insights.isEmpty)
+ #expect(report { $0.mainThread.freezes = 5; $0.session.uptimeSeconds = 300 }.insights.map(\.id) == ["stalls"])
+ // An hour up, but 55 minutes of it asleep: five stalls in five awake minutes are frequent.
+ #expect(report { $0.mainThread.freezes = 5; $0.session.uptimeSeconds = 3600; $0.session.asleepSeconds = 3300 }
+ .insights.map(\.id) == ["stalls"])
+ }
+
+ @Test func aShortIdleOrACheapRecorderSaysNothing() {
+ #expect(report { $0.idle = PerfReport.Idle(samples: 11, cpuPct: 80); $0.selfCost.cpuPct = 0.1 }.insights.isEmpty)
+ #expect(report { $0.idle = PerfReport.Idle(samples: 100, cpuPct: 9) }.insights.isEmpty)
+ }
+
+ @Test func pageFramesAddUpAndIgnoreNonsense() {
+ var f = PerfPageFrames()
+ f.add(PerfPageFrames(longFrames: 2, longestFrameMs: 90, videoFrames: 100, droppedVideoFrames: 3))
+ f.add(PerfPageFrames(longFrames: -1, longestFrameMs: .infinity, videoFrames: 10, droppedVideoFrames: 50))
+ #expect(f == PerfPageFrames(longFrames: 2, longestFrameMs: 90, videoFrames: 110, droppedVideoFrames: 13))
}
@Test func formatting() {
@@ -190,7 +228,10 @@ struct PerfInsightTests {
r.insights = PerfInsights.evaluate(r)
let text = PerfSummary.text(r)
#expect(!text.contains("example.com"))
- #expect(text.contains("Main thread: 2 stalls ≥250 ms, 1 ≥1 s, 0 ≥2 s, longest 1.2 s"))
+ #expect(text.contains("Main thread: 2 freezes ≥250 ms (1 ≥1 s, 0 ≥2 s), longest freeze 1.2 s; longest pass 4.0 s (tracking), 1 outside freezes"))
+ #expect(text.contains("Slow pass sampled: 1.2 s (default), top: 10 main [Cra]"))
+ #expect(text.contains("Video: 12 of 600 frames dropped"))
+ #expect(text.contains("Idle: 3.0 % CPU over 0 min"))
#expect(text.contains("renderer ×3: 12.2 % CPU, 900 MB"))
#expect(text.contains("Previous session: ended without quitting (Cra 0.1.26)"))
#expect(text.contains("Tabs: 40, 6 loaded, 3 asleep, 1 window"))
diff --git a/Sources/Cra/Perf/MainLoopObserver.swift b/Sources/Cra/Perf/MainLoopObserver.swift
index a2eb0ac..7194248 100644
--- a/Sources/Cra/Perf/MainLoopObserver.swift
+++ b/Sources/Cra/Perf/MainLoopObserver.swift
@@ -1,13 +1,16 @@
import Foundation
import os
-/// How long the main thread goes between points where it could take input, measured from the
-/// run loop itself: a CFRunLoopObserver notes the time at each pass and when the loop goes to
-/// sleep, so a pass is never slept-through time. Unlike HitchMonitor it sends the main thread
-/// nothing; the cost is a clock read and an uncontended lock per callback.
+/// How long the main thread works between two points where what it did reaches the screen,
+/// measured from the run loop itself. A pass starts when the loop wakes up and ends where Core
+/// Animation commits: before the loop goes to sleep, or when the loop returns to AppKit (exit),
+/// which then handles the event it returned for, so the next pass starts there. A slept-through
+/// time is never a pass, and an event, its handler, layout and commit are one pass, not several.
+/// Unlike HitchMonitor it sends the main thread nothing; the cost is a clock read and an
+/// uncontended lock per callback.
///
-/// A pass of 2 s or more is seen while it lasts by a watchdog thread (2 Hz), which reports it
-/// from its own thread: the main thread is the one that's stuck.
+/// A long pass is seen while it lasts by a watchdog thread (2 Hz), which reports it from its own
+/// thread (the main thread is the one that's stuck): once at `slowMs`, again at `hangMs`.
final class MainLoopObserver: @unchecked Sendable {
struct Drained {
var passes: LogHistogram
@@ -25,6 +28,7 @@ final class MainLoopObserver: @unchecked Sendable {
var sampledNs: UInt64 = 0
var sampled = 0
var reportedPass: UInt64 = 0
+ var reportedSlow: UInt64 = 0
/// Until the run loop first sleeps: AppKit finishes launching (Cra's bootstrap) inside
/// -[NSApplication run] before the loop's first pass, where no observer sees it.
var launching = true
@@ -35,26 +39,35 @@ final class MainLoopObserver: @unchecked Sendable {
private let shared = OSAllocatedUnfairLock(initialState: Shared())
private var observers: [CFRunLoopObserver] = []
private let stallMs: Double
+ private let slowMs: Double
private let hangMs: Double
private let onStall: (_ ms: Double, _ endedAt: UInt64, _ mode: String) -> Void
+ private let onSlow: (_ ms: Double, _ passStart: UInt64, _ mode: String) -> Void
private let onHang: (_ ms: Double, _ passStart: UInt64, _ mode: String) -> Void
+ private let onWatch: () -> Void
- /// `onStall` runs on the main thread when a pass of `stallMs` or more ends; `onHang` on the
- /// watchdog thread once per pass that reaches `hangMs` while it still lasts.
- init(stallMs: Double, hangMs: Double, onStall: @escaping (Double, UInt64, String) -> Void,
- onHang: @escaping (Double, UInt64, String) -> Void) {
+ /// `onStall` runs on the main thread when a pass of `stallMs` or more ends; `onSlow` and
+ /// `onHang` on the watchdog thread, once per pass that reaches `slowMs` (below `hangMs`) and
+ /// `hangMs` while it still lasts; `onWatch` there at every look (twice a second).
+ init(stallMs: Double, slowMs: Double, hangMs: Double, onStall: @escaping (Double, UInt64, String) -> Void,
+ onSlow: @escaping (Double, UInt64, String) -> Void, onHang: @escaping (Double, UInt64, String) -> Void,
+ onWatch: @escaping () -> Void) {
self.stallMs = stallMs
+ self.slowMs = slowMs
self.hangMs = hangMs
self.onStall = onStall
+ self.onSlow = onSlow
self.onHang = onHang
+ self.onWatch = onWatch
}
func start() {
guard observers.isEmpty else { return }
// A pass starts before anyone else's observer runs and ends after everyone's: Core
- // Animation commits (layout, drawing) in a before-waiting observer, and that is main-thread
- // time too.
- let first: CFRunLoopActivity = [.entry, .afterWaiting, .beforeTimers]
+ // Animation commits (layout, drawing) in a before-waiting and exit observer, and that is
+ // main-thread time too. Entry and before-timers are no boundary: a loop entered or turned
+ // without sleeping or committing is still the same pass (GitHub #135).
+ let first: CFRunLoopActivity = [.afterWaiting]
let last: CFRunLoopActivity = [.beforeWaiting, .exit]
for (activities, order) in [(first, Int.min), (last, Int.max)] {
let o = CFRunLoopObserverCreateWithHandler(nil, activities.rawValue, true, order) { [weak self] _, activity in
@@ -99,13 +112,20 @@ final class MainLoopObserver: @unchecked Sendable {
s.callbacks += 1
var ms: Double?
let launch = s.launching
- if s.passStart != 0 {
- let pass = Double(now &- s.passStart) / 1e6
- s.passes.record(pass)
- if pass >= stallMs { ms = pass }
+ switch activity {
+ case .afterWaiting:
+ if s.passStart == 0 { s.passStart = now }
+ default:
+ // Before waiting or exit: the pass ends. After an exit AppKit goes on to handle
+ // the event the loop returned for, which is the next pass; asleep, nothing is.
+ if s.passStart != 0 {
+ let pass = Double(now &- s.passStart) / 1e6
+ s.passes.record(pass)
+ if pass >= stallMs { ms = pass }
+ }
+ s.passStart = activity == .beforeWaiting ? 0 : now
+ if activity == .beforeWaiting { s.launching = false }
}
- s.passStart = activity == .beforeWaiting ? 0 : now
- if activity == .beforeWaiting { s.launching = false }
// One callback in 256 is timed, for the self-cost estimate.
return (ms, launch, s.callbacks & 255 == 0)
}
@@ -120,16 +140,22 @@ final class MainLoopObserver: @unchecked Sendable {
while true {
usleep(500_000)
let start = Self.threadCPU()
- let (passStart, stopped, reported) = shared.withLock { ($0.passStart, $0.stopped, $0.reportedPass) }
+ let (passStart, stopped, reported, slow, launch) = shared.withLock {
+ ($0.passStart, $0.stopped, $0.reportedPass, $0.reportedSlow, $0.launching)
+ }
if stopped { return }
if passStart != 0, passStart != reported {
let ms = Double(Perf.now() &- passStart) / 1e6
+ let mode = { launch ? "launch" : Self.currentMode() }
if ms >= hangMs {
- shared.withLock { $0.reportedPass = passStart }
- let launch = shared.withLock { $0.launching }
- onHang(ms, passStart, launch ? "launch" : Self.currentMode())
+ shared.withLock { $0.reportedPass = passStart; $0.reportedSlow = passStart }
+ onHang(ms, passStart, mode())
+ } else if ms >= slowMs, passStart != slow {
+ shared.withLock { $0.reportedSlow = passStart }
+ onSlow(ms, passStart, mode())
}
}
+ onWatch()
let cost = Self.threadCPU() &- start
shared.withLock { $0.watchdogNs += cost + 2_000 } // + the wakeup itself, about 2 µs
}
diff --git a/Sources/Cra/Perf/Perf.swift b/Sources/Cra/Perf/Perf.swift
index 106b2cb..eabccf8 100644
--- a/Sources/Cra/Perf/Perf.swift
+++ b/Sources/Cra/Perf/Perf.swift
@@ -14,6 +14,7 @@ enum Perf {
case mark(String, at: UInt64)
case phase(String, ms: Double)
case visibleSite(String?)
+ case app(active: Bool, onScreen: Bool)
}
private static let lock = NSLock()
@@ -21,14 +22,29 @@ enum Perf {
nonisolated(unsafe) private static var early: [Fact]? = []
nonisolated(unsafe) private static var started: [String: UInt64] = [:]
nonisolated(unsafe) private(set) static var launchedAt: UInt64 = now()
+ nonisolated(unsafe) private(set) static var launchedAsleep: UInt64 = asleepNs()
- /// Monotonic nanoseconds (the clock DispatchTime and mach_absolute_time share).
+ /// Monotonic nanoseconds (the clock DispatchTime and mach_absolute_time share). It stops while
+ /// the Mac sleeps, so a sleep is never a long pass.
static func now() -> UInt64 { DispatchTime.now().uptimeNanoseconds }
static func ms(since t: UInt64) -> Double { Double(now() &- t) / 1e6 }
+ /// How long the Mac has slept since it started: the clock that counts sleep less `now()`'s.
+ static func asleepNs() -> UInt64 { clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) &- clock_gettime_nsec_np(CLOCK_UPTIME_RAW) }
+
+ /// Seconds from `launch()` to `t` (a `now()`) on the clock that counts sleep, so the launch's
+ /// wall time plus this is wall time (GitHub #135). `t` is recent: a sleep between `t` and the
+ /// call would be counted before it.
+ static func secondsSinceLaunch(_ t: UInt64) -> Double {
+ Double(Int64(bitPattern: (t &+ asleepNs()) &- (launchedAt &+ launchedAsleep))) / 1e9
+ }
+
/// The top of main.swift: start-up phases count from here.
- static func launch() { launchedAt = now() }
+ static func launch() {
+ launchedAt = now()
+ launchedAsleep = asleepNs()
+ }
/// The recorder, once it starts; nil when recording is off (buffered facts are dropped).
static func attach(_ newSink: ((Fact) -> Void)?) {
@@ -83,6 +99,9 @@ enum Perf {
/// or incognito).
static func visibleSite(_ host: String?) { send(.visibleSite(host)) }
+ /// Whether Cra is the active app and shows a window, for the stalls that happen meanwhile.
+ static func app(active: Bool, onScreen: Bool) { send(.app(active: active, onScreen: onScreen)) }
+
/// A duration that ends on another call: `begin` and `end` with the same key. A begin
/// without an end (a page closed while loading) is forgotten by `cancel` or the next begin.
static func begin(_ metric: PerfMetric, _ key: String) {
diff --git a/Sources/Cra/Perf/PerfCEF.swift b/Sources/Cra/Perf/PerfCEF.swift
index 375e23d..e3f7ea3 100644
--- a/Sources/Cra/Perf/PerfCEF.swift
+++ b/Sources/Cra/Perf/PerfCEF.swift
@@ -2,8 +2,9 @@ import AppKit
import CraCEF
/// The performance recorder's app side, the one Perf file that sees Chromium (lint: perf-boundary):
-/// renderer trouble and failed loads from CraCEF, the visible tab's long tasks from Cra's isolated
-/// world, tab counts from the windows, and the Settings switch.
+/// renderer trouble and failed loads from CraCEF, the visible tab's long tasks and frames from
+/// Cra's isolated world, tab counts from the windows, whether Cra is active and on screen, and
+/// the Settings switch.
@MainActor
enum PerfCEF {
/// From AppDelegate.bootstrapIfNeeded.
@@ -21,15 +22,48 @@ enum PerfCEF {
guard let app else { return }
let start = MainLoopObserver.threadCPU()
recorder.tabs(tabCounts(app))
- if let browser = app.main?.currentBrowser, browser.isCreated, !browser.isClosed { readLongTasks(browser, recorder) }
+ recorder.userIdle(CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: anyInput) >= 60)
+ if let browser = app.main?.currentBrowser, browser.isCreated, !browser.isClosed { readPage(browser, recorder) }
recorder.addMainThreadCost(ns: MainLoopObserver.threadCPU() &- start)
}
+ observeApp(app)
let environment = ProcessInfo.processInfo.environment
observeChanges(of: { [store = app.store] in store.settings.perfRecording }) { on in
PerfRecorder.setRecording(PerfSessionRules.recording(setting: on, environment: environment))
}
}
+ /// kCGAnyInputEventType.
+ private static let anyInput = CGEventType(rawValue: ~0)!
+
+ /// Breadcrumbs for what most often comes right before a stall that no Cra action explains
+ /// (GitHub #135), and the state every stall is tagged with.
+ private static func observeApp(_ app: AppDelegate) {
+ var last: (active: Bool, onScreen: Bool)?
+ let update = { [weak app] in
+ guard let app else { return }
+ let now = (NSApp.isActive, app.mainWindows.contains { $0.window?.occlusionState.contains(.visible) == true })
+ guard last.map({ $0 != now }) ?? true else { return }
+ last = now
+ Perf.app(active: now.0, onScreen: now.1)
+ }
+ let center = NotificationCenter.default
+ let marks: [(Notification.Name, String?)] = [
+ (NSApplication.didBecomeActiveNotification, "app-active"), (NSApplication.didResignActiveNotification, "app-inactive"),
+ (NSWindow.willEnterFullScreenNotification, "fullscreen-enter"), (NSWindow.willExitFullScreenNotification, "fullscreen-exit"),
+ (NSWindow.didChangeOcclusionStateNotification, nil),
+ ]
+ for (name, mark) in marks {
+ center.addObserver(forName: name, object: nil, queue: .main) { _ in
+ MainActor.assumeIsolated {
+ if let mark { HitchMonitor.mark(mark) }
+ update()
+ }
+ }
+ }
+ update()
+ }
+
private static func tabCounts(_ app: AppDelegate) -> PerfTabCounts {
let managers = app.mainWindows.compactMap(\.browsers)
return PerfTabCounts(tabs: app.store.allTabs().count,
@@ -40,23 +74,41 @@ enum PerfCEF {
windows: app.mainWindows.filter { $0.window?.isVisible == true }.count)
}
- /// Reads and resets the counters PageScript keeps in `__cra.lt`. A page without them (Cra's
- /// own pages, one still loading) answers null.
- private static let readLongTasksParams: String = {
- let js = "(() => { const l = window.__cra?.lt; if (!l || !l.n) return null; const r = JSON.stringify([l.n, l.t, l.d]); l.n = 0; l.t = 0; l.d = []; return r; })()"
+ /// Reads and resets the counters PageScript keeps in `__cra.lt` (long tasks) and `__cra.lf`
+ /// (long animation frames), and the frames the page's videos played and dropped since the last
+ /// read (`getVideoPlaybackQuality`, remembered per element). A page without them (Cra's own
+ /// pages, one still loading) or with nothing to say answers null.
+ private static let readPageParams: String = {
+ let js = """
+ (() => { const c = window.__cra, l = c?.lt; if (!l) return null;
+ const f = c.lf || { n: 0, max: 0 }, q = c.vq || (c.vq = new WeakMap()); let vt = 0, vd = 0;
+ for (const v of document.getElementsByTagName('video')) {
+ if (!(v instanceof HTMLVideoElement)) continue;
+ const p = v.getVideoPlaybackQuality(), o = q.get(v) || [0, 0], b = p.totalVideoFrames < o[0] ? [0, 0] : o;
+ vt += p.totalVideoFrames - b[0]; vd += p.droppedVideoFrames - b[1]; q.set(v, [p.totalVideoFrames, p.droppedVideoFrames]);
+ }
+ if (!l.n && !f.n && !vt) return null;
+ const r = JSON.stringify([l.n, l.t, l.d, f.n, f.max, vt, vd]);
+ l.n = 0; l.t = 0; l.d = []; f.n = 0; f.max = 0; return r; })()
+ """
let params: [String: Any] = ["expression": js, "returnByValue": true]
return String(decoding: (try? JSONSerialization.data(withJSONObject: params)) ?? Data(), as: UTF8.self)
}()
- private static func readLongTasks(_ browser: CraBrowser, _ recorder: PerfRecorder) {
- browser.evaluate(inIsolatedWorld: PageScript.world, params: readLongTasksParams) { [weak recorder] result, _ in
+ private static func readPage(_ browser: CraBrowser, _ recorder: PerfRecorder) {
+ browser.evaluate(inIsolatedWorld: PageScript.world, params: readPageParams) { [weak recorder] result, _ in
guard let data = result?.data(using: .utf8),
let json = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any],
let value = (json["result"] as? [String: Any])?["value"] as? String,
- let triple = (try? JSONSerialization.jsonObject(with: Data(value.utf8))) as? [Any], triple.count == 3,
- let count = triple[0] as? Int, let total = triple[1] as? Double else { return }
- let durations = (triple[2] as? [Double]) ?? []
- DispatchQueue.main.async { recorder?.pageLongTasks(count: count, totalMs: total, durations: durations) }
+ let fields = (try? JSONSerialization.jsonObject(with: Data(value.utf8))) as? [Any], fields.count == 7,
+ let count = fields[0] as? Int, let total = fields[1] as? Double else { return }
+ let durations = (fields[2] as? [Double]) ?? []
+ let frames = PerfPageFrames(longFrames: fields[3] as? Int ?? 0, longestFrameMs: fields[4] as? Double ?? 0,
+ videoFrames: fields[5] as? Int ?? 0, droppedVideoFrames: fields[6] as? Int ?? 0)
+ DispatchQueue.main.async {
+ recorder?.pageLongTasks(count: count, totalMs: total, durations: durations)
+ recorder?.pageFrames(frames)
+ }
}
}
}
diff --git a/Sources/Cra/Perf/PerfRecorder.swift b/Sources/Cra/Perf/PerfRecorder.swift
index b548cc9..16e36c5 100644
--- a/Sources/Cra/Perf/PerfRecorder.swift
+++ b/Sources/Cra/Perf/PerfRecorder.swift
@@ -26,7 +26,8 @@ final class PerfRecorder: @unchecked Sendable {
nonisolated(unsafe) private static var appInfo = (cra: "", chromium: "", onboarding: false)
/// Called on the main thread every sample: the app counts tabs and reads the visible page's
- /// long tasks (PerfCEF), then reports back through `tabs(_:)` and `pageLongTasks`.
+ /// long tasks and frames (PerfCEF), then reports back through `tabs(_:)`, `userIdle`,
+ /// `pageLongTasks` and `pageFrames`.
nonisolated(unsafe) static var mainSampler: (@MainActor (PerfRecorder) -> Void)?
let config: Config
@@ -38,7 +39,12 @@ final class PerfRecorder: @unchecked Sendable {
private var state = PerfState()
private var history = PerfHistory()
private var session: PerfSessionInfo
- private let startedAt = Date()
+ /// Cra's start, not the recorder's, cut to the second because the files keep dates to the
+ /// second; `at` counts from there, so `startedAt + at` is the wall time (GitHub #135).
+ private let startedAt: Date
+ private let startedAtFraction: Double
+ /// When this recorder started, for what recording cost per awake second.
+ private let recordingStart = Perf.now()
private var observer: MainLoopObserver?
private var timer: DispatchSourceTimer?
private var processes = PerfProcessTable()
@@ -47,8 +53,20 @@ final class PerfRecorder: @unchecked Sendable {
private var samples = 0
private var hangs = 0
private var hangFiles: [UInt64: URL] = [:]
+ /// `sample` runs this session (at most `config.stacks`), and the pass the slow-pass one is of;
+ /// both under `lock`.
+ private var stacksTaken = 0
+ private var slowPassStart: UInt64?
+ /// The slow pass's `sample` while it runs (the watchdog thread's alone).
+ private var slowSample: Sample?
/// What recording costs, under `lock` (the queue, the watchdog and the main thread add to it).
- private var cost = (cpuNs: UInt64(0), observerNs: UInt64(0), callbacks: 0, bytes: 0, writes: 0)
+ private struct Cost {
+ var observerNs: UInt64 = 0, watchdogNs: UInt64 = 0, processNs: UInt64 = 0, systemNs: UInt64 = 0
+ var filesNs: UInt64 = 0, mainNs: UInt64 = 0
+ var callbacks = 0, bytes = 0, writes = 0
+ var totalNs: UInt64 { observerNs &+ watchdogNs &+ processNs &+ systemNs &+ filesNs &+ mainNs }
+ }
+ private var cost = Cost()
private var stopped = false
// MARK: Lifecycle (main.swift, the settings observer)
@@ -86,6 +104,9 @@ final class PerfRecorder: @unchecked Sendable {
}
init(config: Config, writesFiles: Bool) {
+ let launched = Date().timeIntervalSince1970 - Perf.secondsSinceLaunch(Perf.now())
+ startedAt = Date(timeIntervalSince1970: launched.rounded(.down))
+ startedAtFraction = launched - launched.rounded(.down)
self.config = config
self.writesFiles = writesFiles
files = PerfFiles(folder: config.folder)
@@ -112,9 +133,11 @@ final class PerfRecorder: @unchecked Sendable {
session.firstLaunchAt = history.firstLaunchAt
if session.previous?.abnormal == true { CraLog.note("perf: the previous session (\(session.previous!.id)) ended without quitting") }
- let o = MainLoopObserver(stallMs: config.policy.stallMs, hangMs: config.policy.hangMs,
+ let o = MainLoopObserver(stallMs: config.policy.stallMs, slowMs: config.policy.slowStackMs, hangMs: config.policy.hangMs,
onStall: { [weak self] ms, end, mode in self?.passEnded(ms: ms, at: end, mode: mode) },
- onHang: { [weak self] ms, start, mode in self?.hangStarted(ms: ms, at: start, mode: mode) })
+ onSlow: { [weak self] ms, start, mode in self?.slowPassSeen(ms: ms, at: start, mode: mode) },
+ onHang: { [weak self] ms, start, mode in self?.hangStarted(ms: ms, at: start, mode: mode) },
+ onWatch: { [weak self] in self?.watchSlowSample() })
observer = o
o.start()
let t = DispatchSource.makeTimerSource(queue: queue)
@@ -122,7 +145,7 @@ final class PerfRecorder: @unchecked Sendable {
t.setEventHandler { [weak self] in self?.tick() }
timer = t
t.resume()
- queue.async { [weak self] in self?.measured { self?.sampleProcesses() } }
+ queue.async { [weak self] in self?.sampleProcesses() }
}
private func stop(clean: Bool) {
@@ -146,7 +169,7 @@ final class PerfRecorder: @unchecked Sendable {
// MARK: Facts
- private func seconds(_ at: UInt64) -> Double { Double(Int64(bitPattern: at &- Perf.launchedAt)) / 1e9 }
+ private func seconds(_ at: UInt64) -> Double { Perf.secondsSinceLaunch(at) + startedAtFraction }
private func record(_ fact: Perf.Fact) {
lock.lock()
@@ -157,15 +180,21 @@ final class PerfRecorder: @unchecked Sendable {
case .mark(let label, let at): state.mark(label, at: seconds(at))
case .phase(let name, let ms): state.phase(name, ms: ms)
case .visibleSite(let host): state.visibleSite = host
+ case .app(let active, let onScreen):
+ state.appActive = active
+ state.appOnScreen = onScreen
}
}
/// The main thread, after a pass of a quarter second or more.
private func passEnded(ms: Double, at: UInt64, mode: String) {
- let stall = lock.withLock { state.pass(ms: ms, endedAt: seconds(at), mode: mode) }
+ let start = at &- UInt64(ms * 1e6)
+ let stall: PerfStall? = lock.withLock {
+ if let s = slowPassStart, abs(Int64(bitPattern: s &- start)) < 50_000_000 { state.slowPass?.endedMs = ms }
+ return state.pass(ms: ms, endedAt: seconds(at), mode: mode)
+ }
if let stall, stall.kind != .stall { CraLog.note("perf: main thread blocked \(Int(ms)) ms (\(stall.mode))\(stall.marks.last.map { " after \($0)" } ?? "")") }
// The hang file of this pass learns how it ended.
- let start = at &- UInt64(ms * 1e6)
queue.async { [weak self] in
guard let self, let url = self.hangFiles.first(where: { abs(Int64(bitPattern: $0.key &- start)) < 50_000_000 })?.value,
var hang = self.files.read(PerfHang.self, from: url) else { return }
@@ -177,20 +206,26 @@ final class PerfRecorder: @unchecked Sendable {
/// The watchdog thread, while the main thread is still blocked: the file is written now, in
/// case Cra never comes back (a force quit), then the stack is added.
private func hangStarted(ms: Double, at start: UInt64, mode: String) {
- let (number, marks, site): (Int, [String], String?) = lock.withLock {
+ let (number, marks, site, sampling): (Int, [String], String?, Bool) = lock.withLock {
hangs += 1
- let s = state.policy.stall(at: seconds(Perf.now()), ms: ms, mode: mode, marks: state.marks.elements, site: state.visibleSite)
- return (hangs, s?.marks ?? [], state.visibleSite)
+ let marks = state.policy.breadcrumbs(at: seconds(Perf.now()), ms: ms, marks: state.marks.elements)
+ let sampling = writesFiles && hangs <= 3 && stacksTaken < config.stacks
+ if sampling { stacksTaken += 1 }
+ return (hangs, marks, state.visibleSite, sampling)
}
guard writesFiles, number <= 3 else { return }
let hang = PerfHang(session: sessionID, number: number, at: seconds(start), stalledMs: ms,
mode: PerfStallPolicy.shortMode(mode), marks: marks, site: site,
- stackNote: number <= config.stacks ? "sampling" : "stacks off")
+ stackNote: sampling ? "sampling" : config.stacks == 0 ? "stacks off" : "stack limit reached")
let url = files.hang(session: sessionID, number: number)
write(hang, to: url)
queue.async { [weak self] in self?.hangFiles[start] = url }
- guard number <= config.stacks else { return }
- let stack = Self.sampleMainThread()
+ guard sampling else { return }
+ var stack: [String] = []
+ if let sample = Sample() {
+ while !sample.ended { usleep(100_000) }
+ stack = sample.stack()
+ }
// On the queue, after any `endedMs` the main thread's recovery queued while `sample` ran.
queue.sync {
var latest = files.read(PerfHang.self, from: url) ?? hang
@@ -200,19 +235,94 @@ final class PerfRecorder: @unchecked Sendable {
}
}
- /// One second of `/usr/bin/sample` on Cra, every 10 ms, from the watchdog thread.
- private static func sampleMainThread() -> [String] {
+ /// The watchdog thread, at the first pass of the session that lasts `slowStackMs` (below a
+ /// hang): one `sample` while it still lasts, kept in memory for the report. It doesn't wait
+ /// for the stack, so the watchdog still sees the pass turn into a hang.
+ private func slowPassSeen(ms: Double, at start: UInt64, mode: String) {
+ let take: Bool = lock.withLock {
+ guard slowPassStart == nil, stacksTaken < config.stacks else { return false }
+ stacksTaken += 1
+ slowPassStart = start
+ state.slowPass = PerfHang(session: sessionID, number: 0, at: seconds(start), stalledMs: ms, mode: PerfStallPolicy.shortMode(mode),
+ marks: state.policy.breadcrumbs(at: seconds(Perf.now()), ms: ms, marks: state.marks.elements),
+ site: state.visibleSite, stackNote: "sampling")
+ return true
+ }
+ guard take else { return }
+ slowSample = Sample()
+ if slowSample == nil { lock.withLock { state.slowPass?.stackNote = "sample did not start" } }
+ }
+
+ /// The watchdog thread, twice a second: the slow pass's stack once `sample` has ended.
+ private func watchSlowSample() {
+ guard let sample = slowSample, sample.ended else { return }
+ slowSample = nil
+ let stack = sample.stack()
+ lock.withLock {
+ state.slowPass?.stack = stack
+ state.slowPass?.stackNote = stack.isEmpty ? "sample gave no main thread" : nil
+ }
+ }
+
+ /// One second of `/usr/bin/sample` on Cra, every 10 ms. Started with posix_spawn and checked
+ /// with waitpid rather than Process: inside Cra, Process never learnt that `sample` ended (its
+ /// termination handler never ran and `waitUntilExit` would never return), though it did in
+ /// the test harness, which has no Chromium (GitHub #135).
+ private final class Sample {
+ let pid: pid_t
let out = FileManager.default.temporaryDirectory.appendingPathComponent("cra-hang-\(UUID().uuidString).txt")
- defer { try? FileManager.default.removeItem(at: out) }
- let p = Process()
- p.executableURL = URL(fileURLWithPath: "/usr/bin/sample")
- p.arguments = [String(getpid()), "1", "10", "-mayDie", "-file", out.path]
- p.standardOutput = FileHandle.nullDevice
- p.standardError = FileHandle.nullDevice
- do { try p.run() } catch { return [] }
- p.waitUntilExit()
- guard let text = try? String(contentsOf: out, encoding: .utf8) else { return [] }
- return PerfSampleStack.mainThread(from: text)
+ let started = Perf.now()
+
+ init?() {
+ let args = ["/usr/bin/sample", String(getpid()), "1", "10", "-mayDie", "-file", out.path]
+ var argv: [UnsafeMutablePointer<CChar>?] = args.map { strdup($0) } + [nil]
+ defer { argv.forEach { free($0) } }
+ var actions: posix_spawn_file_actions_t?
+ posix_spawn_file_actions_init(&actions)
+ defer { posix_spawn_file_actions_destroy(&actions) }
+ posix_spawn_file_actions_addopen(&actions, 1, "/dev/null", O_WRONLY, 0)
+ posix_spawn_file_actions_addopen(&actions, 2, "/dev/null", O_WRONLY, 0)
+ // None of Chromium's descriptors go along.
+ var attributes: posix_spawnattr_t?
+ posix_spawnattr_init(&attributes)
+ defer { posix_spawnattr_destroy(&attributes) }
+ posix_spawnattr_setflags(&attributes, Int16(POSIX_SPAWN_CLOEXEC_DEFAULT))
+ var child: pid_t = 0
+ guard posix_spawn(&child, args[0], &actions, &attributes, &argv, environ) == 0 else { return nil }
+ pid = child
+ }
+
+ /// Reaped here, or already by whoever else waits for Cra's children (ECHILD). One that
+ /// takes 20 s is ended: a CI runner's `sample` is slow to attach, never that slow.
+ var ended: Bool {
+ var status: Int32 = 0
+ if waitpid(pid, &status, WNOHANG) != 0 { return true }
+ guard Perf.now() &- started > 20_000_000_000 else { return false }
+ kill(pid, SIGKILL)
+ _ = waitpid(pid, &status, 0)
+ return true
+ }
+
+ private var collected = false
+
+ /// The main thread's stack, once `ended`; the output file goes.
+ func stack() -> [String] {
+ collected = true
+ defer { try? FileManager.default.removeItem(at: out) }
+ return (try? String(contentsOf: out, encoding: .utf8)).map { PerfSampleStack.mainThread(from: $0) } ?? []
+ }
+
+ /// Recording turned off or Cra quitting mid-sample: no zombie, and the raw output (every
+ /// thread, paths included) doesn't stay in the temporary folder.
+ deinit {
+ guard !collected else { return }
+ var status: Int32 = 0
+ if waitpid(pid, &status, WNOHANG) == 0 {
+ kill(pid, SIGKILL)
+ _ = waitpid(pid, &status, 0)
+ }
+ try? FileManager.default.removeItem(at: out)
+ }
}
/// From the main sampler.
@@ -222,14 +332,22 @@ final class PerfRecorder: @unchecked Sendable {
lock.withLock { state.pageLongTasks(count: count, totalMs: totalMs, durations: durations) }
}
- @MainActor func addMainThreadCost(ns: UInt64) { lock.withLock { cost.cpuNs &+= ns } }
+ @MainActor func pageFrames(_ f: PerfPageFrames) { lock.withLock { state.pageFrames(f) } }
+
+ /// The session's slow pass as recorded so far (scripts/test-perf.sh waits for its stack).
+ var slowPass: PerfHang? { lock.withLock { state.slowPass } }
+
+ /// Whether the Mac had no keyboard or mouse input for a minute: the next samples count as idle.
+ @MainActor func userIdle(_ idle: Bool) { lock.withLock { state.userIdle = idle } }
+
+ @MainActor func addMainThreadCost(ns: UInt64) { lock.withLock { cost.mainNs &+= ns } }
// MARK: The queue
private func tick() {
- measured {
- samples += 1
- sampleProcesses()
+ samples += 1
+ sampleProcesses()
+ measured(\.filesNs) {
pullObserver()
history.record(lock.withLock { state.historySample() }, at: Date())
if writesFiles, samples % config.blackBoxEvery == 0 { write(buildReport(), to: files.blackBox) }
@@ -240,19 +358,23 @@ final class PerfRecorder: @unchecked Sendable {
}
}
- private func measured(_ work: () -> Void) {
+ /// The queue's CPU time for `work`, added to one part of the self-cost.
+ private func measured(_ part: WritableKeyPath<Cost, UInt64>, _ work: () -> Void) {
let start = MainLoopObserver.threadCPU()
work()
let used = MainLoopObserver.threadCPU() &- start
- lock.withLock { cost.cpuNs &+= used }
+ lock.withLock { cost[keyPath: part] &+= used }
}
private func sampleProcesses() {
- let groups = processes.update(processSampler.read(), atNs: Perf.now())
- let sys = system.read(dataFolder: config.folder)
+ var groups: [PerfProcessGroup] = []
+ var sys: PerfSystemSample?
+ measured(\.processNs) { groups = processes.update(processSampler.read(), atNs: Perf.now()) }
+ measured(\.systemNs) { sys = system.read(dataFolder: config.folder) }
lock.withLock {
state.processes = groups
state.system = sys
+ state.idleSample(cpuPct: groups.reduce(0) { $0 + $1.cpuPct })
}
}
@@ -260,8 +382,8 @@ final class PerfRecorder: @unchecked Sendable {
guard let d = observer?.drain() else { return }
lock.withLock {
state.addPasses(d.passes)
- cost.cpuNs &+= d.observerNs &+ d.watchdogNs
cost.observerNs &+= d.observerNs
+ cost.watchdogNs &+= d.watchdogNs
cost.callbacks += d.callbacks
}
}
@@ -277,12 +399,18 @@ final class PerfRecorder: @unchecked Sendable {
/// On the queue.
private func buildReport(storage: PerfReport.Storage = PerfReport.Storage()) -> PerfReport {
var info = session
- info.uptimeSeconds = Date().timeIntervalSince(startedAt)
- let uptimeMs = max(info.uptimeSeconds * 1000, 1)
+ let now = Perf.now()
+ info.uptimeSeconds = seconds(now)
+ // Two clock reads apart, a Mac that never slept can come out a few nanoseconds below zero.
+ info.asleepSeconds = max(0, Double(Int64(bitPattern: Perf.asleepNs() &- Perf.launchedAsleep)) / 1e9)
+ // Over the awake time this recorder ran (it can start after launch, from Settings).
+ let recordingMs = max(Double(now &- recordingStart) / 1e6, 1)
let (snapshot, c) = lock.withLock { (state, cost) }
- let selfCost = PerfSelfCost(cpuMs: Double(c.cpuNs) / 1e6, cpuPct: Double(c.cpuNs) / 1e6 / uptimeMs * 100,
- observerCallbacks: c.callbacks, observerMs: Double(c.observerNs) / 1e6,
- bytesWritten: c.bytes, writes: c.writes,
+ func ms(_ ns: UInt64) -> Double { Double(ns) / 1e6 }
+ let selfCost = PerfSelfCost(cpuMs: ms(c.totalNs), cpuPct: ms(c.totalNs) / recordingMs * 100,
+ observerCallbacks: c.callbacks, observerMs: ms(c.observerNs), watchdogMs: ms(c.watchdogNs),
+ processSampleMs: ms(c.processNs), systemSampleMs: ms(c.systemNs), filesMs: ms(c.filesNs),
+ mainSamplerMs: ms(c.mainNs), bytesWritten: c.bytes, writes: c.writes,
memoryBytes: snapshot.estimatedBytes + history.buckets.count * 120)
return snapshot.report(generatedAt: Date(), session: info, selfCost: selfCost, storage: storage)
}
@@ -293,14 +421,15 @@ final class PerfRecorder: @unchecked Sendable {
func snapshot(_ done: @escaping @MainActor (PerfReport) -> Void) {
queue.async { [weak self] in
guard let self else { return }
- self.measured { self.pullObserver() }
+ self.measured(\.filesNs) { self.pullObserver() }
let r = self.buildReport()
DispatchQueue.main.async { done(r) }
}
}
/// Help ▸ Export Diagnostics: `perf.json` (with the profile's size, measured now), the previous
- /// session's black box and the hang files, as text for the zip (Diagnostics redacts them).
+ /// session's black box, the 24-hour history (5-minute buckets: when the stalls and long tasks
+ /// happened) and the hang files, as text for the zip (Diagnostics redacts them).
static func exportFiles(_ done: @escaping @MainActor ([(name: String, text: String)], Set<String>) -> Void) {
let r = shared
let folder = config?.folder
@@ -319,8 +448,11 @@ final class PerfRecorder: @unchecked Sendable {
}
if let folder {
let files = PerfFiles(folder: folder)
- for url in [files.previousBlackBox] + files.hangFiles() {
- if let data = try? Data(contentsOf: url) { out.append(("perf/" + url.lastPathComponent, String(decoding: data, as: UTF8.self))) }
+ for url in [files.previousBlackBox, files.history] + files.hangFiles() {
+ guard let data = try? Data(contentsOf: url) else { continue }
+ out.append(("perf/" + url.lastPathComponent, String(decoding: data, as: UTF8.self)))
+ // The last session's sites too, whatever its schema, for "Hide site names".
+ sites.formUnion(Self.sites(in: (try? JSONSerialization.jsonObject(with: data)) ?? [:]))
}
}
DispatchQueue.main.async { done(out, sites) }
@@ -328,6 +460,16 @@ final class PerfRecorder: @unchecked Sendable {
if let r { r.queue.async(execute: work) } else { DispatchQueue.global(qos: .utility).async(execute: work) }
}
+ /// Every `site` value in a Perf file.
+ private static func sites(in json: Any) -> Set<String> {
+ if let d = json as? [String: Any] {
+ return d.reduce(into: Set<String>()) { s, kv in
+ if kv.key == "site", let host = kv.value as? String { s.insert(host) } else { s.formUnion(sites(in: kv.value)) }
+ }
+ }
+ return (json as? [Any])?.reduce(into: Set<String>()) { $0.formUnion(sites(in: $1)) } ?? []
+ }
+
/// Settings ▸ Performance ▸ Clear: the files, and what this session recorded so far.
static func clear() {
if let r = shared {
diff --git a/Sources/Cra/Perf/PerformancePane.swift b/Sources/Cra/Perf/PerformancePane.swift
index 3b6fd5c..7dcc6b7 100644
--- a/Sources/Cra/Perf/PerformancePane.swift
+++ b/Sources/Cra/Perf/PerformancePane.swift
@@ -76,9 +76,11 @@ struct PerformancePane: View {
}
let m = r.mainThread
card("Main thread", [
- ("Stalls of ¼ s or more", "\(m.stalls)"),
- ("Freezes of 1 s or more", "\(m.longStalls)"),
- ("Longest", m.longestMs > 0 ? PerfInsights.duration(m.longestMs) : "–"),
+ ("Stalls of ¼ s or more", "\(m.freezes)"),
+ ("Freezes of 1 s or more", "\(m.longFreezes)"),
+ ("Longest freeze", m.longestFreezeMs > 0 ? PerfInsights.duration(m.longestFreezeMs) : "–"),
+ ("In menus, drags, alerts, start-up", "\(m.outsideFreezes)"),
+ ("Longest pass", m.longestPassMs > 0 ? PerfInsights.duration(m.longestPassMs) + (m.longestPassMode.map { " (\($0))" } ?? "") : "–"),
("Typical pass (99 %)", PerfInsights.duration(m.passes.percentile(0.99))),
])
card("Processes", r.processes.filter { $0.count > 0 }.map { g in
@@ -114,6 +116,8 @@ struct PerformancePane: View {
rows.append((title, "\(PerfInsights.duration(h.percentile(0.5))) typical · \(PerfInsights.duration(h.percentile(0.95))) slow"))
}
if r.tabs.longTasks > 0 { rows.append(("Busy page (long tasks)", "\(r.tabs.longTasks) · \(PerfInsights.duration(r.tabs.longTaskMs))")) }
+ if r.tabs.longFrames > 0 { rows.append(("Late frames", "\(r.tabs.longFrames) · longest \(PerfInsights.duration(r.tabs.longestFrameMs))")) }
+ if r.tabs.videoFrames > 0 { rows.append(("Video frames dropped", "\(r.tabs.droppedVideoFrames) of \(r.tabs.videoFrames)")) }
return rows
}
diff --git a/Sources/Cra/Services/BrowserManager.swift b/Sources/Cra/Services/BrowserManager.swift
index 2a987b8..39ade0e 100644
--- a/Sources/Cra/Services/BrowserManager.swift
+++ b/Sources/Cra/Services/BrowserManager.swift
@@ -757,6 +757,7 @@ final class BrowserManager: NSObject, @preconcurrency CraBrowserDelegate {
func browser(_ browser: CraBrowser, didChangeLoading loading: Bool, canGoBack: Bool, canGoForward: Bool) {
guard let id = tabID(of: browser) else { return }
if loading { runtimes[id]?.navigationRevision &+= 1 }
+ HitchMonitor.mark(loading ? "load-start" : "load-end")
if loading { Perf.begin(.pageLoad, id.uuidString) } else {
Perf.end(.pageLoad, id.uuidString)
Perf.end(.tabWake, id.uuidString)
diff --git a/Sources/Cra/Services/Diagnostics.swift b/Sources/Cra/Services/Diagnostics.swift
index 5422f1e..9407709 100644
--- a/Sources/Cra/Services/Diagnostics.swift
+++ b/Sources/Cra/Services/Diagnostics.swift
@@ -13,6 +13,7 @@ enum Diagnostics {
/// Asks where to save, unless `destination` is given (cra://debug?exportDiagnostics=<path>).
static func export(windows: [MainWindowController], store: SidebarStore, to destination: URL? = nil) {
CraLog.note("diagnostics export started")
+ HitchMonitor.mark("diagnostics-export")
collectPages(windows: windows, store: store) { pages in
CraCEFRuntime.sendBrowserDevToolsMethod("SystemInfo.getInfo", params: "{}") { gpu, error in
// The profile's size is measured off the main thread.
diff --git a/Sources/Cra/Services/PageScript.swift b/Sources/Cra/Services/PageScript.swift
index e19e07c..7fb6509 100644
--- a/Sources/Cra/Services/PageScript.swift
+++ b/Sources/Cra/Services/PageScript.swift
@@ -56,6 +56,10 @@ enum PageScript {
new PerformanceObserver(list => {
for (const e of list.getEntries()) { lt.n++; lt.t += e.duration; if (lt.d.length < 100) lt.d.push(Math.round(e.duration)); }
}).observe({ type: 'longtask' });
+ const lf = cra.lf = { n: 0, max: 0 };
+ new PerformanceObserver(list => {
+ for (const e of list.getEntries()) { lf.n++; if (e.duration > lf.max) lf.max = Math.round(e.duration); }
+ }).observe({ type: 'long-animation-frame' });
} catch {}
})();
"""
diff --git a/Tests/PerfTests.swift b/Tests/PerfTests.swift
index 9c9b2ac..3c05987 100644
--- a/Tests/PerfTests.swift
+++ b/Tests/PerfTests.swift
@@ -18,15 +18,18 @@ enum PerfTests {
recordingOffStopsSamplingAndWriting()
theFacadeWaitsForTheRecorderAndTimesWork()
aSlowLaunchIsAStallButNotAFreeze()
+ workAcrossATimerBoundaryIsOnePass()
+ theFirstSlowPassGetsAStack()
print("Perf recorder: \(checks) checks passed")
}
- static func config(_ name: String, hangMs: Double = 2000, stacks: Int = 0) -> PerfRecorder.Config {
+ static func config(_ name: String, hangMs: Double = 2000, slowMs: Double = 1000, stacks: Int = 0) -> PerfRecorder.Config {
var c = PerfRecorder.Config(folder: root.appendingPathComponent(name))
c.sampleInterval = 0.2
c.blackBoxEvery = 2
c.historyEvery = 3
c.policy.hangMs = hangMs
+ c.policy.slowStackMs = slowMs
c.stacks = stacks
return c
}
@@ -63,20 +66,29 @@ enum PerfTests {
let c = config("stall")
start(c)
spin(0.3)
+ var stallEnded = Date.distantPast
DispatchQueue.main.async {
Perf.mark("cra://debug?stall=500")
usleep(500_000)
+ stallEnded = Date()
}
spin(0.5)
DispatchQueue.main.async { usleep(300_000) }
spin(1.2)
guard let r = report() else { return expect(false, "a report while recording") }
- expect(r.mainThread.stalls == 2, "both stalls counted (\(r.mainThread.stalls))")
+ expect(r.mainThread.freezes == 2, "both stalls counted (\(r.mainThread.freezes))")
+ expect(r.mainThread.longestPassMs >= 500 && r.mainThread.longestPassMode == "default", "the longest pass (\(r.mainThread.longestPassMs))")
expect(r.mainThread.recent.allSatisfy { $0.mode != "launch" }, "a quick start is no stall")
let first = r.mainThread.recent.first
expect(first.map { $0.ms >= 500 && $0.ms < 900 } == true, "the injected 500 ms stall measured as such (\(first?.ms ?? 0))")
expect(first?.marks == ["cra://debug"], "with its breadcrumb, cut before the query (\(first?.marks ?? []))")
expect(first?.mode == "default" && first?.freeze == true, "in the default mode: a freeze")
+ // Read back from the file, which keeps dates to the second.
+ let file = (try? PerfReport.encoder().encode(r)).flatMap { try? PerfReport.decoder.decode(PerfReport.self, from: $0) }
+ let drift = file.flatMap { f in f.mainThread.recent.first.map { f.session.startedAt.addingTimeInterval($0.at).timeIntervalSince(stallEnded) } } ?? 99
+ expect(abs(drift) < 0.1, "startedAt + at in perf.json is the wall time the stall ended (off by \(drift) s)")
+ expect(r.session.uptimeSeconds >= 2 && r.session.uptimeSeconds < 60 && r.session.asleepSeconds < 1,
+ "uptime from Cra's start on the same clock (\(r.session.uptimeSeconds) s)")
expect(r.mainThread.passes.count > 2 && r.mainThread.passes.maxMs >= 500, "every pass in the histogram")
expect(r.processes.contains { $0.kind == .browser && $0.count == 1 && $0.footprintBytes > 0 }, "libproc read this process")
expect(r.system?.memoryPressure != nil && r.system?.thermalState.isEmpty == false, "the Mac sampled")
@@ -90,7 +102,8 @@ enum PerfTests {
var sites: Set<String> = ["unset"]
PerfRecorder.exportFiles { exported = $0; sites = $1 }
for _ in 0..<100 where exported.isEmpty { spin(0.05) }
- expect(exported.first?.name == "perf.json" && exported.first!.text.contains("\"stalls\" : 2"), "perf.json in the export")
+ expect(exported.first?.name == "perf.json" && exported.first!.text.contains("\"freezes\" : 2"), "perf.json in the export")
+ expect(exported.contains { $0.name == "perf/history.json" }, "with the 24-hour history")
expect(sites.isEmpty, "no sites: no page was on screen")
PerfRecorder.quit()
@@ -152,12 +165,19 @@ enum PerfTests {
files.write(marker!, to: files.marker)
files.write(PerfHang(session: id!, number: 1, at: 1, stalledMs: 3000, mode: "default", marks: [], site: nil), to: files.hang(session: id!, number: 1))
files.write(PerfHang(session: "older000", number: 1, at: 1, stalledMs: 3000, mode: "default", marks: [], site: nil), to: files.hang(session: "older000", number: 1))
+ files.write(PerfHang(session: id!, number: 2, at: 1, stalledMs: 3000, mode: "default", marks: [], site: "old.example.com"),
+ to: files.hang(session: id!, number: 2))
start(c)
let previous = report()?.session.previous
expect(previous?.abnormal == true && previous?.id == id, "the next launch reports it as ended without quitting")
expect(report()?.insights.first?.id == "previous-abnormal", "and says so first")
- expect(files.hangFiles().map(\.lastPathComponent) == ["hang-\(id!)-1.json"], "its hang file stays, older ones go")
+ expect(files.hangFiles().map(\.lastPathComponent) == ["hang-\(id!)-1.json", "hang-\(id!)-2.json"], "its hang files stay, older ones go")
+ var sites: Set<String> = []
+ var exported = false
+ PerfRecorder.exportFiles { _, s in sites = s; exported = true }
+ for _ in 0..<100 where !exported { spin(0.05) }
+ expect(sites.contains("old.example.com"), "the last session's sites are hidden with this one's (\(sites))")
PerfRecorder.quit()
}
@@ -229,9 +249,54 @@ enum PerfTests {
let r = report()
expect(r?.mainThread.recent.first?.mode == "launch" && r?.mainThread.recent.first?.freeze == false,
"a slow start-up is a stall in the launch mode (\(r?.mainThread.recent.first?.mode ?? "none"))")
- expect(r?.mainThread.stalls == 0 && r?.mainThread.outsideFreezes == 1
+ expect(r?.mainThread.freezes == 0 && r?.mainThread.outsideFreezes == 1
&& r?.insights.contains { $0.id == "freezes" || $0.id == "stalls" } == false,
- "not a freeze, no freeze finding (\(r?.mainThread.stalls ?? -1), \(r?.mainThread.outsideFreezes ?? -1), \(r?.insights.map(\.id) ?? []))")
+ "not a freeze, no freeze finding (\(r?.mainThread.freezes ?? -1), \(r?.mainThread.outsideFreezes ?? -1), \(r?.insights.map(\.id) ?? []))")
+ expect(r?.mainThread.longestPassMs ?? 0 >= 400 && r?.mainThread.longestPassMode == "launch"
+ && r?.insights.contains { $0.id == "outside-freezes" } == true,
+ "but the longest pass, and a finding that says it was not counted (\(r?.mainThread.longestPassMs ?? -1))")
+ PerfRecorder.quit()
+ }
+
+ /// GitHub #135: a timer and a queued block that run in one turn of the loop, with no sleep
+ /// or commit between them, are one 300 ms pass. Split at before-timers they were two 150 ms
+ /// halves and no stall at all.
+ @MainActor static func workAcrossATimerBoundaryIsOnePass() {
+ start(config("one-pass"))
+ spin(0.5)
+ let before = report()?.mainThread.passes.count ?? 0
+ Timer.scheduledTimer(withTimeInterval: 0, repeats: false) { _ in usleep(150_000) }
+ DispatchQueue.main.async { usleep(150_000) }
+ spin(0.6)
+ let r = report()
+ let stalls = r?.mainThread.recent ?? []
+ expect(stalls.count == 1 && stalls[0].ms >= 300 && stalls[0].ms < 600 && r?.mainThread.freezes == 1,
+ "one stall of 300 ms (\(stalls.map(\.ms)))")
+ // A quiet loop sleeps between the 5 Hz samples and reports: a few passes a second, not hundreds.
+ let after = r?.mainThread.passes.count ?? 0
+ expect(after - before < 60, "one pass per wake, not one per observer callback (\(after - before) in 0.6 s)")
+ expect(r.map { $0.selfCost.observerCallbacks < 4 * $0.mainThread.passes.count + 20 } == true,
+ "two or three callbacks a pass (\(r?.selfCost.observerCallbacks ?? -1) for \(r?.mainThread.passes.count ?? -1))")
+ PerfRecorder.quit()
+ }
+
+ /// GitHub #135: the first pass that lasts `slowStackMs` gets one `sample`, in memory, while the
+ /// watchdog goes on watching; the report says how long the pass was in the end.
+ @MainActor static func theFirstSlowPassGetsAStack() {
+ start(config("slow", hangMs: 30_000, slowMs: 400, stacks: 1))
+ spin(0.3)
+ DispatchQueue.main.async {
+ Perf.mark("slow-test")
+ busyMainThreadForPerfTest(atLeast: 0.8, limit: 15) { PerfRecorder.shared?.slowPass.map { $0.stackNote != "sampling" } ?? false }
+ }
+ spin(0.5)
+ let slow = report()?.mainThread.slowPass
+ expect(slow?.stack.contains { $0.contains("busyMainThreadForPerfTest") } == true,
+ "with the main thread's stack (\(slow?.stackNote ?? "no slow pass")): \(slow?.stack.prefix(6) ?? [])")
+ expect(slow.map { $0.stalledMs >= 400 && ($0.endedMs ?? 0) >= 800 && $0.marks == ["slow-test"] && $0.number == 0 } == true,
+ "how long it was when sampled and in the end, after what (\(slow?.stalledMs ?? -1), \(slow?.endedMs ?? -1))")
+ expect(FileManager.default.fileExists(atPath: PerfFiles(folder: config("slow").folder).hangFiles().first?.path ?? "/nonexistent") == false,
+ "no hang file below the hang threshold")
PerfRecorder.quit()
}
diff --git a/docs/performance.md b/docs/performance.md
index 9ac5419..3bd9b8c 100644
--- a/docs/performance.md
+++ b/docs/performance.md
@@ -29,30 +29,58 @@ libproc calls; once every 5 s it counts tabs and asks the visible page for its l
## What a pass is
`MainLoopObserver` puts two `CFRunLoopObserver`s on the main run loop in the common modes: one
-first in line when a pass starts (entry, after waiting, before timers), one last in line when
-it ends (before waiting, exit), so Core Animation's commit counts. A pass is the time between
-two points where the main thread could take input. It differs from HitchMonitor, which times
-how long a block queued on the main queue waits; that wait can span several passes. On
-`fifty-tabs` both see the same number of 250 ms stalls, give or take one.
-
-- 250 ms or more is a stall, 1 s a freeze, 2 s a hang (the watchdog writes `hang-<session>-<n>.json`
- while it lasts, then runs `/usr/bin/sample` on Cra for a second, at most three times a session).
-- Stalls are tagged with the run-loop mode. `tracking` (a held menu, a drag), `modal` (an
- alert) and `launch` (from the recorder's start until the run loop first sleeps: AppKit runs
- `applicationDidFinishLaunching` before the loop's first pass) are recorded but are not freezes.
-- Breadcrumbs are HitchMonitor's marks (swipes, page shown, extension popups, scripted
- commands), cut before any `?`.
+first in line when the loop wakes up (after waiting), one last in line before it sleeps (before
+waiting) and when it returns to AppKit (exit). Those last two are where Core Animation commits,
+so a pass is the main thread's work between two points where what it did reaches the screen:
+from waking (or from the exit before it, after which AppKit handles the event the loop returned
+for) to the next sleep or exit, commit included. Entering the loop and turning it without
+sleeping are no boundary (GitHub #135): before, a pass ended at every observer callback, so one
+turn was recorded as two halves, and a 300 ms turn of a 150 ms timer and a 150 ms block was no
+stall at all. A pass differs from HitchMonitor's numbers, which time how long a block queued on
+the main queue waits; that wait can span several passes.
+
+- 250 ms or more is a stall. A stall in the default mode is a freeze: `freezes`, `longFreezes`
+ (1 s or more) and `hangs` (2 s or more) count those, and `longestFreezeMs` is the longest.
+- `tracking` (a held menu, a drag), `modal` (an alert, a save panel) and `launch` (from the
+ recorder's start until the run loop first sleeps: AppKit runs `applicationDidFinishLaunching`
+ before the loop's first pass) are loops Cra is in on purpose: their stalls are in
+ `outsideFreezes`, not in the freeze counts. `longestPassMs` and `longestPassMode` are over every
+ pass, the honest answer to "how long was the main thread blocked". Copy Summary and the pane
+ show both.
+- Stacks: at 2 s the watchdog writes `hang-<session>-<n>.json` while the hang lasts, then runs
+ `/usr/bin/sample` on Cra for a second. No pass in the first real exports reached 2 s, so the
+ session's first pass that is still going at 1 s (`slowStackMs`) gets one `sample` too, kept in
+ memory as `mainThread.slowPass` (no file: it is no hang). At most three `sample` runs a session
+ in all. `sample` takes a second to attach, so a pass that ends soon after shows mostly the idle
+ run loop; `endedMs` against `stalledMs` says how much of the sample saw the block.
+- Breadcrumbs are HitchMonitor's marks, cut before any `?`: swipes, page shown, page loads
+ starting and ending (`load-start`, `load-end`), extension popups, scripted commands, Cra
+ becoming active or not (`app-active`, `app-inactive`), full screen (`fullscreen-enter`,
+ `fullscreen-exit`) and `diagnostics-export`. Every stall also says whether Cra was the active
+ app and whether a window of it was on screen (`active`, `onScreen`).
+- Time: `session.startedAt` is when Cra started (the top of `main.swift`), cut to the second as
+ the files keep dates, and every `at` (stalls, marks, events, hang files) is seconds since then
+ on the clock that counts the Mac's sleep, so `startedAt + at` lines up with `cra.log`. `uptimeSeconds` is on the
+ same clock and `asleepSeconds` says how much of it the Mac slept; pass durations use the clock
+ that stops in sleep, so a sleep is never a stall.
## What is recorded
-Main-thread passes as a histogram, stalls with mode, breadcrumbs and the site on screen. Per
-process kind (browser, renderer, GPU, utility, other, from Chromium's `--type=` argument): CPU,
-memory footprint, disk reads and writes, wakeups (in the file only), and processes started and
-ended. The Mac: CPU, memory pressure, thermal state, Low Power Mode, free disk. Tabs by state,
-sleep refusals by reason, and timings for tab switch, page create, page load and waking a
-slept tab. Long tasks in the visible page. Renderer crashes, hangs and failed loads with their
-codes. Start-up phases, JSON load and save times, the profile's and caches' size (at export).
-The session: a random id, the launch number, the previous session and whether it quit.
+Main-thread passes as a histogram, stalls with mode, breadcrumbs, the site on screen and
+whether Cra was active. Per process kind (browser, renderer, GPU, utility, other, from
+Chromium's `--type=` argument): CPU, memory footprint, disk reads and writes, wakeups (in the
+file only), and processes started and ended. The Mac: CPU, memory pressure, thermal state, Low
+Power Mode, free disk, and Cra's average CPU while nobody used the Mac (`idle`: 5-second samples
+with no keyboard or mouse input for a minute). Tabs by state, sleep refusals by reason, and
+timings for tab switch, page create, page load and waking a slept tab. In the visible page,
+read with the long tasks every 5 s and never in background tabs: long tasks, Chromium's long
+animation frames (`longFrames`, `longestFrameMs`: a passive observer, it asks for no frames) and
+the frames its videos played and dropped (`getVideoPlaybackQuality`). Renderer crashes, hangs
+and failed loads with their codes. Start-up phases, JSON load and save times (saves run on the
+persistence queue, so `jsonSave` is never a main-thread stall), the profile's and caches' size
+(at export only). The session: a random id, the launch number, the previous session and whether
+it quit. The totals in `tabs` are since launch; `history.json`, in the export too, has stalls
+and long tasks per 5 minutes.
Never: addresses beyond the host, paths, query strings, titles, page text, form values, typed
text, an installation id. Incognito pages give counts only.
@@ -74,17 +102,29 @@ A `session.open` found at launch means that session ended without quitting (cras
`kill -9`, power loss). It is made only after `CraCEFRuntime.initialize` succeeds, so the
"another Cra is running" and failed-start exits never leave one.
-## Measured (2 Nov 2026, M-series Mac)
+## Measured (3 Nov 2026, M-series Mac)
| | Budget | Measured |
|---|---|---|
-| CPU | < 0.1 % of a core | 0.027–0.039 % (`fifty-tabs`, three runs; an idle session 0.027 %) |
-| Main thread | < 0.2 ms per tick | ~95 ns per observer callback (114,663 callbacks, 10.8 ms over a `fifty-tabs` run) |
+| CPU | < 0.1 % of a core | 0.029 % (`fifty-tabs`; 2 Nov, before #135: 0.027–0.039 %) |
+| Main thread | < 0.2 ms per tick | ~110 ns per observer callback (60,319 callbacks, 6.7 ms over a `fifty-tabs` run; 114,663 before #135) |
| Memory | < 5 MB | ~5 KB of recorder data (`selfCost.memoryBytes`) |
| Disk writes | < 60 KB per minute | ~5 KB per minute (black box) plus ~4 KB per minute (history) |
-`selfCost` in every report says what recording cost that session; `scripts/scenario.sh`
-fails a scenario above 0.1 %.
+`selfCost` in every report says what recording cost that session, over the awake time it ran,
+and which part: `observerMs`, `watchdogMs`, `processSampleMs` (libproc over every helper),
+`systemSampleMs`, `filesMs` (black box and history: building, encoding, writing) and
+`mainSamplerMs` (the tab count and the visible page's read on the main thread).
+`scripts/scenario.sh` fails a scenario above 0.1 %, and a report above it says so in its
+findings (`profiler-overhead`); the numbers still hold, the finding is there so a report says
+what recording cost.
+
+The first real exports (GitHub #135: 2 500 tabs, 9 loaded, 3 hours) measured 0.13–0.145 %,
+against 0.03 % on `fifty-tabs`. Most of it was the 5-second queue, not the observer (536 ms of
+6 342), and the observer's callbacks were twice what they needed to be (one record per
+callback). The callbacks are halved (#135); the parts above say where the rest goes on the next
+real export. On `fifty-tabs` the 13.5 ms were: observer 6.7, process sample 5.1 (libproc and the
+`--type=` of each new renderer), main sampler 1.0, watchdog 0.4, system sample 0.3, files 0.1.
## Spikes (phase 0)
@@ -128,7 +168,8 @@ fails a scenario above 0.1 %.
`Services/HitchMonitor.swift`, `Services/Diagnostics.swift`, `UI/Settings/SettingsWindowController.swift`
(the pane, `isOpen`). Then the two rules in `scripts/lint.py`.
3. `CraLog.sitesSeen` and `SiteNames` serve "Hide site names"; keep them if that stays.
-4. The `lt` observer in `PageScript.isolatedWorld`, and `setProcessEventHandler` in CraCEF.
+4. The `lt` and `lf` observers in `PageScript.isolatedWorld`, the `load-start`/`load-end` and
+ `diagnostics-export` marks, and `setProcessEventHandler` in CraCEF.
5. `perfRecording` and `perfHidesSiteNames` in `AppSettings` (old files keep loading: unknown
keys are ignored) and their lines in `Fixtures/Model`.
6. The harness lists that compile `Perf.swift` and `CraCore/Perf/*.swift`
diff --git a/scripts/scenario.sh b/scripts/scenario.sh
index fc9dae0..16cc21c 100755
--- a/scripts/scenario.sh
+++ b/scripts/scenario.sh
@@ -216,7 +216,7 @@ import json, sys
r = json.load(open(sys.argv[1]))
cpu, budget, hitch = r["selfCost"]["cpuPct"], float(sys.argv[2]), int(sys.argv[3])
m = r["mainThread"]
-seen = m["stalls"] + m["outsideFreezes"]
+seen = m["freezes"] + m["outsideFreezes"]
print(f"perf: recording costs {cpu:.3f} % of a core (budget {budget}); stalls ≥250 ms: recorder {seen}, HitchMonitor {hitch}")
if cpu > budget: print(f"PROBLEM the performance recorder costs {cpu:.3f} % of a core, budget {budget} %")
# This run's HitchMonitor log starts after the launch the recorder also saw; a few either way is timing.