Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
import CoreML
import Foundation
import Security
import XCTest
#if arch(arm64)
import FluidAudio
#endif
/// Diagnostic, not an accuracy gate: nonempty ASR output is measured, never asserted away.
/// Synthetic input only; no fixture audio, devices, downloader, or production pipeline used.
@MainActor
final class MeetingReliabilityNegativeScreenTests: XCTestCase {
func testSyntheticNoiseIsDeterministicBoundedAndNonzero() {
let a = ReliabilitySyntheticInput.samples(count: 16_000, noise: true, seed: 42)
XCTAssertEqual(a, ReliabilitySyntheticInput.samples(count: 16_000, noise: true, seed: 42))
XCTAssertNotEqual(a, ReliabilitySyntheticInput.samples(count: 16_000, noise: true, seed: 43))
XCTAssertTrue(a.allSatisfy { $0.isFinite && abs($0) <= 0.001 })
XCTAssertGreaterThan(a.reduce(0.0) { $0 + Double($1 * $1) }, 0)
XCTAssertEqual(ReliabilitySyntheticInput.samples(count: 100, noise: false, seed: 42), [Float](repeating: 0, count: 100))
}
func testOutputCountingAndExposureAccounting() {
XCTAssertEqual(ReliabilitySyntheticInput.wordCount(" \n\t"), 0)
XCTAssertEqual(ReliabilitySyntheticInput.wordCount("Okay, I\nget it."), 4)
XCTAssertEqual(ReliabilitySyntheticInput.segmentLengths(totalSeconds: 61, segmentSeconds: 60), [60, 1])
XCTAssertEqual(ReliabilitySyntheticInput.segmentLengths(totalSeconds: 600, segmentSeconds: 3).reduce(0, +), 600)
}
func testCanonicalArtifactRelativePathHandlesTemporaryDirectoryAliases() throws {
let fm = FileManager.default
let fixture = fm.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true)
try fm.createDirectory(at: fixture, withIntermediateDirectories: true)
defer { try? fm.removeItem(at: fixture) }
let root = fixture.appendingPathComponent("model", isDirectory: true)
try fm.createDirectory(at: root, withIntermediateDirectories: true)
let child = root.appendingPathComponent("weight.bin")
try Data([0]).write(to: child)
let alias = fixture.appendingPathComponent("alias", isDirectory: true)
try fm.createSymbolicLink(at: alias, withDestinationURL: root)
XCTAssertEqual(try ReliabilitySyntheticInput.relativeArtifactPath(child, under: alias), "weight.bin")
XCTAssertThrowsError(try ReliabilitySyntheticInput.relativeArtifactPath(fixture.appendingPathComponent("outside"), under: root))
}
func testSyntheticSilenceAndLowNoiseNegativeScreen() async throws {
guard ProcessInfo.processInfo.environment["FLUID_RELIABILITY_NEGATIVE_SCREEN"] == "1" else {
throw XCTSkip("opt-in: FLUID_RELIABILITY_NEGATIVE_SCREEN=1; default 600 audio seconds per condition/mode")
}
#if arch(arm64)
try self.validateHost()
let env = ProcessInfo.processInfo.environment
let seconds = try boundedInteger(env["FLUID_RELIABILITY_SECONDS"], default: 600, range: 60...600)
let budgetSeconds = try boundedInteger(env["FLUID_RELIABILITY_WALL_SECONDS"], default: 1200, range: 60...1800)
guard let root = BaselineInputLoader.inputsRoot() else { throw ScreenError.invalidInputs }
let manifest = try BaselineInputLoader.loadManifest(from: root)
let manifestHash = try BaselineHashing.sha256Hex(ofFileAt: root.appendingPathComponent("manifest.json"))
var report = NegativeScreenReport(
manifestSHA256: manifestHash,
modelRepository: manifest.modelRepository,
requestedAudioSecondsPerConditionMode: seconds,
wallBudgetSeconds: budgetSeconds
)
defer { attach(report) }
var activeManager: AsrManager?
do {
report.phase = "artifact_validation"
let repository = try validateArtifacts(root: root, manifest: manifest)
// No AsrModels.load/download API: even load(from:) can enter DownloadUtils.
// Direct CoreML loading has no model retrieval fallback.
let config = MLModelConfiguration()
config.computeUnits = .cpuAndNeuralEngine
let preprocessorConfig = MLModelConfiguration()
preprocessorConfig.computeUnits = .cpuOnly
report.phase = "vocabulary_validation"
let vocabData = try Data(contentsOf: repository.appendingPathComponent("parakeet_vocab.json"))
let rawVocab = try JSONDecoder().decode([String: String].self, from: vocabData)
var vocabulary: [Int: String] = [:]
for (key, value) in rawVocab {
guard let index = Int(key), vocabulary[index] == nil else { throw ScreenError.invalidVocabulary }
vocabulary[index] = value
}
guard !vocabulary.isEmpty else { throw ScreenError.invalidVocabulary }
report.phase = "local_model_load"
let loadStart = Date()
let models = try AsrModels(
encoder: MLModel(contentsOf: repository.appendingPathComponent("Encoder.mlmodelc"), configuration: config),
preprocessor: MLModel(contentsOf: repository.appendingPathComponent("Preprocessor.mlmodelc"), configuration: preprocessorConfig),
decoder: MLModel(contentsOf: repository.appendingPathComponent("Decoder.mlmodelc"), configuration: config),
joint: MLModel(contentsOf: repository.appendingPathComponent("JointDecision.mlmodelc"), configuration: config),
configuration: config,
vocabulary: vocabulary,
version: .v2
)
let manager = AsrManager()
activeManager = manager
try await manager.initialize(models: models)
report.modelLoadSeconds = Date().timeIntervalSince(loadStart)
report.phase = "synthetic_inference"
let clock = ContinuousClock()
let deadline = clock.now + .seconds(budgetSeconds)
// One manager, strictly sequential awaits. Pinned AsrManager.transcribe([Float])
// awaits throwing resetDecoderState before returning (3fd6388, lines 524–548).
// Noise is the same continuous seeded sequence under either segmentation mode.
for noise in [false, true] {
let allSamples = ReliabilitySyntheticInput.samples(count: seconds * 16_000, noise: noise, seed: 0x46565030)
for segmentSeconds in [60, 3] {
var offset = 0
for length in ReliabilitySyntheticInput.segmentLengths(totalSeconds: seconds, segmentSeconds: segmentSeconds) {
try Task.checkCancellation()
guard clock.now < deadline else { throw ScreenError.wallBudgetExceeded }
let count = length * 16_000
let samples = Array(allSamples[offset..<(offset + count)])
let start = clock.now
let result = try await manager.transcribe(samples, source: .microphone)
let elapsed = start.duration(to: clock.now).components
report.runs.append(.init(
condition: noise ? "synthetic_uniform_noise_peak_0.001" : "digital_silence",
mode: segmentSeconds == 60 ? "whole_chunk_60s" : "short_turn_3s",
startSample: offset,
sampleCount: count,
nonempty: !result.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty,
whitespaceWordCount: ReliabilitySyntheticInput.wordCount(result.text),
latencySeconds: Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18
))
offset += count
}
}
}
await manager.cleanup()
activeManager = nil
report.status = "complete"
report.phase = "complete"
} catch {
if let activeManager { await activeManager.cleanup() }
// Only error type, never ASR text or potentially sensitive file/error payload.
report.status = "aborted_\(String(describing: type(of: error)))"
if let safeError = error as? ScreenError { report.failureCode = String(describing: safeError) }
throw error
}
#else
throw XCTSkip("requires Apple Silicon")
#endif
}
private func boundedInteger(_ raw: String?, default fallback: Int, range: ClosedRange<Int>) throws -> Int {
guard let raw else { return fallback }
guard let value = Int(raw), range.contains(value) else { throw ScreenError.invalidConfiguration }
return value
}
private func validateHost() throws {
guard Bundle.main.bundleIdentifier == "com.FluidApp.ASRBaselineHost",
let task = SecTaskCreateFromSelf(nil),
(SecTaskCopyValueForEntitlement(task, "com.apple.security.app-sandbox" as CFString, nil) as? Bool) == true
else { throw ScreenError.unsafeHost }
for key in ["com.apple.security.network.client", "com.apple.security.network.server", "com.apple.security.device.audio-input"] {
guard (SecTaskCopyValueForEntitlement(task, key as CFString, nil) as? Bool) != true else { throw ScreenError.unsafeHost }
}
}
private func validateArtifacts(root: URL, manifest: ASRBaselineManifest) throws -> URL {
guard manifest.modelRepository == "parakeet-tdt-0.6b-v2-coreml", !manifest.artifacts.isEmpty else { throw ScreenError.invalidRepository }
let paths = manifest.artifacts.map(\.path)
guard Set(paths).count == paths.count else { throw ScreenError.duplicateArtifactPaths }
let repository = try BaselinePathSafety.validatedURL(forRelativePath: manifest.modelRepository, under: root)
for artifact in manifest.artifacts {
guard artifact.path.hasPrefix(manifest.modelRepository + "/") else { throw ScreenError.invalidArtifactPath }
let url = try BaselinePathSafety.validatedURL(forRelativePath: artifact.path, under: root)
guard try BaselineHashing.sha256Hex(ofFileAt: url) == artifact.sha256 else { throw ScreenError.artifactHashMismatch }
}
// Reject unmanifested files too: a partial manifest cannot bless an unverified model.
guard let enumerator = FileManager.default.enumerator(at: repository, includingPropertiesForKeys: [.isRegularFileKey, .isSymbolicLinkKey])
else { throw ScreenError.artifactEnumerationFailed }
var actual = Set<String>()
for case let url as URL in enumerator {
let values = try url.resourceValues(forKeys: [.isRegularFileKey, .isSymbolicLinkKey])
guard values.isSymbolicLink != true else { throw ScreenError.symbolicLinkArtifact }
if values.isRegularFile == true {
try actual.insert(manifest.modelRepository + "/" + (ReliabilitySyntheticInput.relativeArtifactPath(url, under: repository)))
}
}
guard actual == Set(paths) else { throw ScreenError.artifactSetMismatch }
return repository
}
private func attach(_ report: NegativeScreenReport) {
let encoder = JSONEncoder()
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
guard let data = try? encoder.encode(report) else { XCTFail("negative screen report encoding failed"); return }
let attachment = XCTAttachment(data: data, uniformTypeIdentifier: "public.json")
attachment.name = "MeetingReliabilityNegativeScreenReport"
attachment.lifetime = .keepAlways
add(attachment)
}
}
private nonisolated enum ScreenError: Error {
case invalidInputs, invalidConfiguration, unsafeHost, wallBudgetExceeded
case invalidRepository, duplicateArtifactPaths, invalidArtifactPath, artifactHashMismatch
case artifactEnumerationFailed, symbolicLinkArtifact, artifactSetMismatch, invalidVocabulary
}
nonisolated enum ReliabilitySyntheticInput {
static func relativeArtifactPath(_ child: URL, under root: URL) throws -> String {
// FileManager may enumerate /private/tmp while resolvingSymlinksInPath returns
// /tmp. Never trim using the character count of a differently normalized URL.
let rootParts = BaselinePathSafety.canonical(root).pathComponents
let childParts = BaselinePathSafety.canonical(child).pathComponents
guard childParts.count > rootParts.count,
Array(childParts.prefix(rootParts.count)) == rootParts else { throw ScreenError.invalidArtifactPath }
return childParts.dropFirst(rootParts.count).joined(separator: "/")
}
static func samples(count: Int, noise: Bool, seed: UInt64) -> [Float] {
guard noise else { return [Float](repeating: 0, count: count) }
var state = seed
return (0..<count).map { _ in
state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407
return (Float(state >> 40) / Float(0xffffff) * 2 - 1) * 0.001
}
}
static func wordCount(_ text: String) -> Int { text.split(whereSeparator: { $0.isWhitespace }).count }
static func segmentLengths(totalSeconds: Int, segmentSeconds: Int) -> [Int] {
precondition(totalSeconds >= 0 && segmentSeconds > 0)
return stride(from: 0, to: totalSeconds, by: segmentSeconds).map { min(segmentSeconds, totalSeconds - $0) }
}
}
private nonisolated struct NegativeScreenReport: Encodable {
let schemaVersion = 1
let kind = "synthetic_negative_asr_screen_not_meeting_pipeline"
let verdict = "measured_only"
let resetSemantics = "Pinned AsrManager array transcribe awaits resetDecoderState before successful return; errors abort the screen. Not the streaming EOU decoder."
let timestamp = ISO8601DateFormatter().string(from: Date())
let sampleRateHz = 16_000
let channels = 1
let seed = "0x46565030; LCG64; top24-bit uniform; peak=0.001 (~-60dBFS)"
let modelProvenance = "Preexisting host-bundled ASRBaselineInputs; exact artifact-set SHA256 verification against manifest; manifest is integrity provenance, not independent publisher authentication"
let networkPolicy = "Direct local CoreML constructors; no downloader; sandbox required and network/audio-input entitlements forbidden"
// Keep the exact fixture text or diagnostic output together for comparison.
// swiftlint:disable:next line_length
let scope = "No recorded audio read. No VAD, AEC, diarization, speaker profiles, production provider, or live recognizer. Noise is not AEC residual. Whitespace-delimited word counts, not linguistic token accuracy. Repeated silence is deterministic exposure, not independent statistical trials. No emptiness assertion. Wall budget checked between calls; a non-cooperative CoreML call may exceed it."
let notTested = ["real AEC residual", "quiet speech recall", "short spoken acknowledgments", "double talk", "playback leakage", "caption reset recovery"]
let manifestSHA256: String
let modelRepository: String
let requestedAudioSecondsPerConditionMode: Int
let wallBudgetSeconds: Int
var modelLoadSeconds: Double?
var status = "in_progress"
var phase = "initialization"
var failureCode: String?
var runs: [Run] = []
struct Run: Encodable {
let condition: String
let mode: String
let startSample: Int
let sampleCount: Int
let nonempty: Bool
let whitespaceWordCount: Int
let latencySeconds: Double
var audioSeconds: Double { Double(self.sampleCount) / 16_000 }
enum CodingKeys: CodingKey { case condition, mode, startSample, sampleCount, nonempty, whitespaceWordCount, latencySeconds, audioSeconds }
func encode(to encoder: Encoder) throws {
var c = encoder.container(keyedBy: CodingKeys.self)
try c.encode(self.condition, forKey: .condition); try c.encode(self.mode, forKey: .mode)
try c.encode(self.startSample, forKey: .startSample); try c.encode(self.sampleCount, forKey: .sampleCount)
try c.encode(self.nonempty, forKey: .nonempty); try c.encode(self.whitespaceWordCount, forKey: .whitespaceWordCount)
try c.encode(self.latencySeconds, forKey: .latencySeconds); try c.encode(self.audioSeconds, forKey: .audioSeconds)
}
}
}