Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

preserve dictionary word weights

altic-devc14b760eccd5
diff --git a/Sources/Fluid/Services/DictionaryTransferService.swift b/Sources/Fluid/Services/DictionaryTransferService.swift
index 0fa9b83c..a9f5a185 100644
--- a/Sources/Fluid/Services/DictionaryTransferService.swift
+++ b/Sources/Fluid/Services/DictionaryTransferService.swift
@@ -63,11 +63,18 @@ struct DictionaryTransferReplacement: Codable, Equatable {
struct DictionaryTransferDocument: Codable, Equatable {
let replacements: [DictionaryTransferReplacement]
- let customWords: [String]
+ let customWords: [DictionaryTransferCustomWord]
init(replacements: [DictionaryTransferReplacement], customWords: [String]) {
+ self.init(
+ replacements: replacements,
+ customWordEntries: customWords.map { DictionaryTransferCustomWord(text: $0, weight: nil) }
+ )
+ }
+
+ init(replacements: [DictionaryTransferReplacement], customWordEntries: [DictionaryTransferCustomWord]) {
self.replacements = replacements
- self.customWords = customWords
+ self.customWords = customWordEntries
}
init(from decoder: Decoder) throws {
@@ -123,7 +130,7 @@ struct DictionaryTransferDocument: Codable, Equatable {
private static func decodeCustomWords(
from container: KeyedDecodingContainer<CodingKeys>
- ) throws -> (found: Bool, values: [String]) {
+ ) throws -> (found: Bool, values: [DictionaryTransferCustomWord]) {
let customWords = try Self.decodeCustomWordValues(from: container, key: .customWords)
if customWords.found {
return customWords
@@ -150,27 +157,48 @@ struct DictionaryTransferDocument: Codable, Equatable {
private static func decodeCustomWordValues(
from container: KeyedDecodingContainer<CodingKeys>,
key: CodingKeys
- ) throws -> (found: Bool, values: [String]) {
+ ) throws -> (found: Bool, values: [DictionaryTransferCustomWord]) {
guard container.contains(key) else { return (false, []) }
- return try (true, container.decode([DictionaryTransferCustomWord].self, forKey: key).map(\.text))
+ return try (true, container.decode([DictionaryTransferCustomWord].self, forKey: key))
}
}
-private struct DictionaryTransferCustomWord: Decodable {
+struct DictionaryTransferCustomWord: Codable, Equatable {
let text: String
+ let weight: Float?
+
+ init(text: String, weight: Float?) {
+ self.text = text
+ self.weight = weight
+ }
init(from decoder: Decoder) throws {
if let text = try? decoder.singleValueContainer().decode(String.self) {
self.text = text
+ self.weight = nil
return
}
let container = try decoder.container(keyedBy: CodingKeys.self)
self.text = try container.decode(String.self, forKey: .text)
+ self.weight = try container.decodeIfPresent(Float.self, forKey: .weight)
+ }
+
+ func encode(to encoder: Encoder) throws {
+ if self.weight == nil {
+ var container = encoder.singleValueContainer()
+ try container.encode(self.text)
+ return
+ }
+
+ var container = encoder.container(keyedBy: CodingKeys.self)
+ try container.encode(self.text, forKey: .text)
+ try container.encodeIfPresent(self.weight, forKey: .weight)
}
private enum CodingKeys: String, CodingKey {
case text
+ case weight
}
}
@@ -212,10 +240,7 @@ final class DictionaryTransferService {
func makeExportDocument() throws -> DictionaryTransferDocument {
try DictionaryTransferDocument(
replacements: SettingsStore.shared.customDictionaryEntries.compactMap(Self.exportReplacement(from:)),
- customWords: Self.normalizedUniqueStrings(
- ParakeetVocabularyStore.shared.loadUserBoostTerms().map(\.text),
- lowercased: false
- )
+ customWordEntries: Self.exportCustomWords(from: ParakeetVocabularyStore.shared.loadUserBoostTerms())
)
}
@@ -276,13 +301,13 @@ final class DictionaryTransferService {
var customWords = mode == .replace ? [] : currentCustomWords
for word in normalizedDocument.customWords {
guard customWords.count < self.maxCustomWords else { break }
- if customWords.contains(where: { $0.text.caseInsensitiveCompare(word) == .orderedSame }) {
+ if customWords.contains(where: { $0.text.caseInsensitiveCompare(word.text) == .orderedSame }) {
continue
}
customWords.append(
ParakeetVocabularyStore.VocabularyConfig.Term(
- text: word,
- weight: self.importedCustomWordWeight,
+ text: word.text,
+ weight: word.weight ?? self.importedCustomWordWeight,
aliases: []
)
)
@@ -294,7 +319,7 @@ final class DictionaryTransferService {
private static func normalizedDocument(_ document: DictionaryTransferDocument) -> DictionaryTransferDocument {
DictionaryTransferDocument(
replacements: document.replacements.compactMap(self.exportReplacement(from:)),
- customWords: self.normalizedUniqueStrings(document.customWords, lowercased: false)
+ customWordEntries: self.exportCustomWords(from: document.customWords)
)
}
@@ -321,6 +346,33 @@ final class DictionaryTransferService {
return SettingsStore.CustomDictionaryEntry(triggers: from, replacement: to)
}
+ private static func exportCustomWords(
+ from terms: [ParakeetVocabularyStore.VocabularyConfig.Term]
+ ) -> [DictionaryTransferCustomWord] {
+ self.exportCustomWords(
+ from: terms.map { DictionaryTransferCustomWord(text: $0.text, weight: $0.weight) }
+ )
+ }
+
+ private static func exportCustomWords(
+ from words: [DictionaryTransferCustomWord]
+ ) -> [DictionaryTransferCustomWord] {
+ var seen: Set<String> = []
+ var result: [DictionaryTransferCustomWord] = []
+ result.reserveCapacity(words.count)
+
+ for word in words {
+ let text = word.text.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard !text.isEmpty else { continue }
+ let key = text.lowercased()
+ guard !seen.contains(key) else { continue }
+ seen.insert(key)
+ result.append(DictionaryTransferCustomWord(text: text, weight: word.weight))
+ }
+
+ return result
+ }
+
private static func upsert(
_ entry: SettingsStore.CustomDictionaryEntry,
into entries: inout [SettingsStore.CustomDictionaryEntry]
diff --git a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
index 87f45ec0..fecf613f 100644
--- a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
+++ b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
@@ -113,6 +113,7 @@ final class DictationE2ETests: XCTestCase {
XCTAssertEqual(state.replacements.first?.replacement, "FluidVoice")
XCTAssertEqual(state.customWords.map(\.text), ["FluidVoice", "Barath"])
XCTAssertEqual(state.customWords.map(\.weight), [10.0, 10.0])
+ XCTAssertEqual(state.customWords.map(\.aliases), [[], []])
}
func testDictionaryTransferImport_mergeDedupesAndMovesDuplicateTriggers() throws {
@@ -295,7 +296,8 @@ final class DictationE2ETests: XCTestCase {
XCTAssertEqual(state.replacements.count, 0)
XCTAssertEqual(state.customWords.map(\.text), ["FluidVoice", "GEMBA-E"])
- XCTAssertEqual(state.customWords.map(\.weight), [10.0, 10.0])
+ XCTAssertEqual(state.customWords.map(\.weight), [13.0, 10.0])
+ XCTAssertEqual(state.customWords.map(\.aliases), [[], []])
}
func testDictionaryTransferImport_acceptsLocalAPICustomWordsResponse() throws {
@@ -325,6 +327,8 @@ final class DictationE2ETests: XCTestCase {
XCTAssertEqual(state.replacements.count, 0)
XCTAssertEqual(state.customWords.map(\.text), ["FluidVoice", "Barath"])
+ XCTAssertEqual(state.customWords.map(\.weight), [10.0, 10.0])
+ XCTAssertEqual(state.customWords.map(\.aliases), [[], []])
}
func testDictationEndToEnd_whisperTiny_transcribesFixture() async throws {
@@ -804,7 +808,7 @@ final class DictationE2ETests: XCTestCase {
}
func testLooksLikeHTML_rejectsLeadingWhitespaceAndBOMVariants() {
- let bom: [UInt8] = [0xEF, 0xBB, 0xBF]
+ let bom: [UInt8] = [0xef, 0xbb, 0xbf]
// Leading ASCII whitespace before the markup token.
XCTAssertTrue(HuggingFaceModelDownloader.looksLikeHTML(Data(" \n\t<!DOCTYPE html>".utf8)))
@@ -828,9 +832,9 @@ final class DictationE2ETests: XCTestCase {
// MIL program text (`model.mil`).
XCTAssertFalse(HuggingFaceModelDownloader.looksLikeHTML(Data("program(1.0)\n[buildInfo = ...]".utf8)))
// Binary CoreML / Mach-O magic prefix.
- XCTAssertFalse(HuggingFaceModelDownloader.looksLikeHTML(Data([0xCF, 0xFA, 0xED, 0xFE, 0x07, 0x00])))
+ XCTAssertFalse(HuggingFaceModelDownloader.looksLikeHTML(Data([0xcf, 0xfa, 0xed, 0xfe, 0x07, 0x00])))
// Leading-NUL binary (e.g. coremldata.bin / weight.bin style payloads).
- XCTAssertFalse(HuggingFaceModelDownloader.looksLikeHTML(Data([0x00, 0x00, 0x01, 0x3C, 0x68])))
+ XCTAssertFalse(HuggingFaceModelDownloader.looksLikeHTML(Data([0x00, 0x00, 0x01, 0x3c, 0x68])))
// Empty payload.
XCTAssertFalse(HuggingFaceModelDownloader.looksLikeHTML(Data()))
// A stray `<` NOT followed by a markup-ish byte must not be over-rejected.
@@ -915,7 +919,7 @@ final class DictationE2ETests: XCTestCase {
// A loose required file (e.g. a tokenizer) with real binary content.
let tokenizerURL = root.appendingPathComponent("tokenizer.model")
- try Data([0x0A, 0x09, 0x05, 0x00]).write(to: tokenizerURL)
+ try Data([0x0a, 0x09, 0x05, 0x00]).write(to: tokenizerURL)
let entries = [packageName, "tokenizer.model"]
@@ -939,7 +943,7 @@ final class DictationE2ETests: XCTestCase {
// Missing entries and an empty required directory are conservative: never flagged corrupt
// on uncertainty (incompleteness is the existence check's concern, not this one's).
- try Data([0x0A, 0x09, 0x05, 0x00]).write(to: tokenizerURL)
+ try Data([0x0a, 0x09, 0x05, 0x00]).write(to: tokenizerURL)
let emptyPackage = root.appendingPathComponent("empty.mlpackage", isDirectory: true)
try FileManager.default.createDirectory(at: emptyPackage, withIntermediateDirectories: true)
XCTAssertFalse(