Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: add letter-case flags to `WordMetadata` (#1578)
Andrew Dunbar569d6162f01b
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex b3a2b9723..cdf311a21 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -18,6 +18,7 @@ use harper_core::linting::{LintGroup, Linter};use harper_core::parsers::{Markdown, MarkdownOptions, OrgMode, PlainEnglish};use harper_core::{CharStringExt, Dialect, Document, TokenKind, TokenStringExt, WordMetadata, remove_overlaps,+ word_metadata_orthography::OrthFlags,};use harper_literate_haskell::LiterateHaskellParser;use harper_pos_utils::{BrillChunker, BrillTagger};@@ -136,6 +137,9 @@ enum Args {/// Emit a decompressed, line-separated list of the compounds in Harper's dictionary./// As long as there's either an open or hyphenated spelling.Compounds,+ /// Emit a decompressed, line-separated list of the words in Harper's dictionary+ /// which occur in more than one lettercase variant.+ CaseVariants,/// Provided a sentence or phrase, emit a list of each noun phrase contained within.NominalPhrases { input: String },}@@ -666,6 +670,37 @@ fn main() -> anyhow::Result<()> {println!("\nFound {} compound word groups", results.len());Ok(())}+ Args::CaseVariants => {+ let case_bitmask = OrthFlags::LOWERCASE+ | OrthFlags::TITLECASE+ | OrthFlags::ALLCAPS+ | OrthFlags::LOWER_CAMEL+ | OrthFlags::UPPER_CAMEL;+ let mut processed_words = HashMap::new();+ let mut longest_word = 0;+ for word in dictionary.words_iter() {+ if let Some(metadata) = dictionary.get_word_metadata(word) {+ let orth = metadata.orth_info;+ let bits = orth.bits() & case_bitmask.bits();++ if bits.count_ones() > 1 {+ longest_word = longest_word.max(word.len());+ // Mask out all bits except the case-related ones before printing+ processed_words.insert(+ word.to_string(),+ OrthFlags::from_bits_truncate(orth.bits() & case_bitmask.bits()),+ );+ }+ }+ }+ let mut processed_words: Vec<_> = processed_words.into_iter().collect();+ processed_words.sort_by_key(|(word, _)| word.clone());+ let longest_num = (processed_words.len() - 1).to_string().len();+ for (i, (word, orth)) in processed_words.iter().enumerate() {+ println!("{i:>longest_num$} {word:<longest_word$} : {orth:?}");+ }+ Ok(())+ }Args::NominalPhrases { input } => {let doc = Document::new_markdown_default_curated(&input);diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex a679df844..9fcd8bafd 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -31606,7 +31606,8 @@ leisureliness/Ngleisurewear/Ngleitmotif/NgSleitmotiv/NgS-lemma/~NS+lemma/~N0S+lemmata/N9lemme/zGlemming/Nglemon/~NwJVSgdiff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 063d78672..2bf20f67b 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -26,7 +26,8 @@ mod token;mod token_kind;mod token_string_ext;mod vec_ext;-mod word_metadata;+pub mod word_metadata;+pub mod word_metadata_orthography;use render_markdown::render_markdown;use std::collections::VecDeque;diff --git a/harper-core/src/spell/rune/attribute_list.rs b/harper-core/src/spell/rune/attribute_list.rsindex e2c0d3655..439b65669 100644--- a/harper-core/src/spell/rune/attribute_list.rs+++ b/harper-core/src/spell/rune/attribute_list.rs@@ -13,6 +13,7 @@ use super::expansion::{};use super::word_list::MarkedWord;use crate::spell::WordId;+use crate::word_metadata_orthography::OrthFlags;use crate::{CharString, Span, WordMetadata};#[derive(Debug, Clone)]@@ -50,6 +51,9 @@ impl AttributeList {dest.reserve(word.attributes.len() + 1);let mut gifted_metadata = WordMetadata::default();+ let orth_flags = check_orthography(&word);+ gifted_metadata.orth_info = orth_flags;+let mut conditional_expansion_metadata = Vec::new();for attr in &word.attributes {@@ -229,6 +233,218 @@ impl AttributeList {}}+/// Gather metadata about the orthography of a word.+fn check_orthography(word: &MarkedWord) -> OrthFlags {+ use crate::char_ext::CharExt;+ use crate::word_metadata_orthography::OrthFlags;++ let mut ortho_flags = OrthFlags::default();+ let mut all_lower = true;+ let mut all_upper = true;+ let mut first_is_upper = false;+ let mut first_is_lower = false;+ let mut saw_upper_after_first = false;+ let mut saw_lower_after_first = false;+ let mut is_first_char = true;+ let mut upper_to_lower = false;+ let mut lower_to_upper = false;+ let letter_count = word+ .letters+ .iter()+ .filter(|c| c.is_english_lingual())+ .count();++ for &c in &word.letters {+ // Multiword: contains at least one space+ if c == ' ' {+ ortho_flags |= OrthFlags::MULTIWORD;+ continue;+ }+ // Hyphenated: contains at least one hyphen+ if c == '-' {+ ortho_flags |= OrthFlags::HYPHENATED;+ continue;+ }+ // Apostrophe: contains at least one apostrophe (straight or curly)+ if c == '\'' || c == '’' {+ ortho_flags |= OrthFlags::APOSTROPHE;+ continue;+ }+ // Only consider English letters for case flags+ if !c.is_english_lingual() {+ continue;+ }+ if c.is_lowercase() {+ all_upper = false;+ if is_first_char {+ first_is_lower = true;+ } else {+ saw_lower_after_first = true;+ if upper_to_lower {+ lower_to_upper = true;+ }+ upper_to_lower = true;+ }+ } else if c.is_uppercase() {+ all_lower = false;+ if is_first_char {+ first_is_upper = true;+ } else {+ saw_upper_after_first = true;+ if lower_to_upper {+ upper_to_lower = true;+ }+ lower_to_upper = true;+ }+ } else {+ // Non-cased char (e.g., numbers, symbols) - ignore for case flags+ // Reset case tracking after non-letter character+ first_is_upper = false;+ first_is_lower = false;+ upper_to_lower = false;+ lower_to_upper = false;+ }+ is_first_char = false;+ }++ // Set case-related orthography flags+ if letter_count > 0 {+ if all_lower {+ ortho_flags |= OrthFlags::LOWERCASE;+ }+ if all_upper {+ ortho_flags |= OrthFlags::ALLCAPS;+ }+ // Only mark as TITLECASE if more than one letter+ if letter_count > 1 && first_is_upper && !saw_upper_after_first {+ ortho_flags |= OrthFlags::TITLECASE;+ }+ // LowerCamel: first is lowercase and there's at least one uppercase character after it+ // Note: This must come after Titlecase check to avoid marking Titlecase words as LowerCamel+ // Example: "pH" is LowerCamel, but "Providence" is Titlecase+ if first_is_lower && saw_upper_after_first {+ ortho_flags |= OrthFlags::LOWER_CAMEL;+ }+ // UpperCamel: first is uppercase and there are both lowercase and uppercase characters after it+ // Note: This must come after Titlecase check to avoid marking Titlecase words as UpperCamel+ // Example: "CamelCase" is UpperCamel, but "Providence" is Titlecase+ if first_is_upper && saw_lower_after_first && saw_upper_after_first {+ ortho_flags |= OrthFlags::UPPER_CAMEL;+ }+ }++ ortho_flags+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::word_metadata_orthography::OrthFlags;++ fn check_orthography_str(s: &str) -> OrthFlags {+ // let word = MarkedWord::new(s.chars().collect());+ let word = MarkedWord {+ letters: s.chars().collect(),+ attributes: Vec::new(),+ };+ check_orthography(&word)+ }++ #[test]+ fn test_lowercase() {+ let flags = check_orthography_str("hello");+ assert!(flags.contains(OrthFlags::LOWERCASE));+ assert!(!flags.contains(OrthFlags::TITLECASE));+ assert!(!flags.contains(OrthFlags::ALLCAPS));+ assert!(!flags.contains(OrthFlags::LOWER_CAMEL));+ assert!(!flags.contains(OrthFlags::UPPER_CAMEL));++ // With non-letters+ let flags = check_orthography_str("hello123");+ assert!(flags.contains(OrthFlags::LOWERCASE));+ }++ #[test]+ fn test_titlecase() {+ let flags = check_orthography_str("Hello");+ assert!(!flags.contains(OrthFlags::LOWERCASE));+ assert!(flags.contains(OrthFlags::TITLECASE));+ assert!(!flags.contains(OrthFlags::ALLCAPS));+ assert!(!flags.contains(OrthFlags::LOWER_CAMEL));+ assert!(!flags.contains(OrthFlags::UPPER_CAMEL));++ // Examples that should be titlecase+ assert!(check_orthography_str("World").contains(OrthFlags::TITLECASE));+ assert!(check_orthography_str("Something").contains(OrthFlags::TITLECASE));++ // These examples should NOT be titlecase (they're UPPER_CAMEL)+ assert!(!check_orthography_str("McDonald").contains(OrthFlags::TITLECASE));+ assert!(!check_orthography_str("O'Reilly").contains(OrthFlags::TITLECASE));++ // Single character should not be titlecase+ assert!(!check_orthography_str("A").contains(OrthFlags::TITLECASE));+ }++ #[test]+ fn test_allcaps() {+ let flags = check_orthography_str("HELLO");+ assert!(!flags.contains(OrthFlags::LOWERCASE));+ assert!(!flags.contains(OrthFlags::TITLECASE));+ assert!(flags.contains(OrthFlags::ALLCAPS));+ assert!(!flags.contains(OrthFlags::LOWER_CAMEL));+ assert!(!flags.contains(OrthFlags::UPPER_CAMEL));++ // Examples from docs+ assert!(check_orthography_str("NASA").contains(OrthFlags::ALLCAPS));+ assert!(check_orthography_str("I").contains(OrthFlags::ALLCAPS));+ }++ #[test]+ fn test_lower_camel() {+ let flags = check_orthography_str("helloWorld");+ assert!(!flags.contains(OrthFlags::LOWERCASE));+ assert!(!flags.contains(OrthFlags::TITLECASE));+ assert!(!flags.contains(OrthFlags::ALLCAPS));+ assert!(flags.contains(OrthFlags::LOWER_CAMEL));+ assert!(!flags.contains(OrthFlags::UPPER_CAMEL));++ // Examples from docs+ assert!(check_orthography_str("getHTTPResponse").contains(OrthFlags::LOWER_CAMEL));+ assert!(check_orthography_str("eBay").contains(OrthFlags::LOWER_CAMEL));++ // All lowercase should not be lower camel+ assert!(!check_orthography_str("hello").contains(OrthFlags::LOWER_CAMEL));++ // Starts with uppercase should not be lower camel+ assert!(!check_orthography_str("HelloWorld").contains(OrthFlags::LOWER_CAMEL));+ }++ #[test]+ fn test_upper_camel() {+ let flags = check_orthography_str("HelloWorld");+ assert!(!flags.contains(OrthFlags::LOWERCASE));+ assert!(!flags.contains(OrthFlags::TITLECASE));+ assert!(!flags.contains(OrthFlags::ALLCAPS));+ assert!(!flags.contains(OrthFlags::LOWER_CAMEL));+ assert!(flags.contains(OrthFlags::UPPER_CAMEL));++ // Examples from docs+ assert!(check_orthography_str("HttpRequest").contains(OrthFlags::UPPER_CAMEL));+ assert!(check_orthography_str("McDonald").contains(OrthFlags::UPPER_CAMEL));+ assert!(check_orthography_str("O'Reilly").contains(OrthFlags::UPPER_CAMEL));+ assert!(check_orthography_str("XMLHttpRequest").contains(OrthFlags::UPPER_CAMEL));++ // Titlecase should not be upper camel+ assert!(!check_orthography_str("Hello").contains(OrthFlags::UPPER_CAMEL));++ // All caps should not be upper camel+ assert!(!check_orthography_str("NASA").contains(OrthFlags::UPPER_CAMEL));++ // Needs at least 3 chars+ assert!(!check_orthography_str("Hi").contains(OrthFlags::UPPER_CAMEL));+ }+}+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct HumanReadableAttributeList {affixes: HashMap<char, HumanReadableExpansion>,diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rsindex 213a75777..cc32fe0b4 100644--- a/harper-core/src/token_kind.rs+++ b/harper-core/src/token_kind.rs@@ -115,7 +115,14 @@ impl TokenKind {// Generic word methodsis_swear,- is_likely_homograph+ is_likely_homograph,++ // Orthography methods+ is_lowercase,+ is_titlecase,+ is_allcaps,+ is_lower_camel,+ is_upper_camel}// Word metadata delegation methods not generated by macrodiff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rsindex 735430b4f..60065ea5e 100644--- a/harper-core/src/word_metadata.rs+++ b/harper-core/src/word_metadata.rs@@ -10,8 +10,12 @@ use strum_macros::{Display, EnumCount, EnumString, VariantArray};use std::convert::TryFrom;use crate::spell::WordId;+use crate::word_metadata_orthography::OrthFlags;use crate::{Document, TokenKind, TokenStringExt};+/// This represents a "lexeme" or "headword" which is case-folded but affix-expanded.+/// So not only lemmata but also inflected forms are stored here, with "horn" and "horns" each+/// having their own lexeme, but "Ivy" and "ivy" share the same lexeme.#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Hash)]pub struct WordMetadata {pub noun: Option<NounData>,@@ -26,6 +30,9 @@ pub struct WordMetadata {/// valid in all dialects of English.#[serde(default = "default_default")]pub dialects: DialectFlags,+ /// Orthographic information: letter case, spaces, hyphens, etc.+ #[serde(default = "OrthFlags::empty")]+ pub orth_info: OrthFlags,/// Whether the word is a [determiner](https://en.wikipedia.org/wiki/English_determiners).pub determiner: Option<DeterminerData>,/// Whether the word is a [preposition](https://www.merriam-webster.com/dictionary/preposition).@@ -180,6 +187,7 @@ impl WordMetadata {adverb: merge!(self.adverb, other.adverb),conjunction: merge!(self.conjunction, other.conjunction),dialects: self.dialects | other.dialects,+ orth_info: self.orth_info | other.orth_info,swear: self.swear.or(other.swear),determiner: merge!(self.determiner, other.determiner),preposition: self.preposition || other.preposition,@@ -624,6 +632,74 @@ impl WordMetadata {matches!(self.swear, Some(true))}+ // Orthographic queries++ /// Does the lexeme for this word cover an all-lowercase variant? (e.g., "hello")+ ///+ /// This returns true if all letters in the word are lowercase. Words containing+ /// non-letter characters (like numbers or symbols) are only considered if all+ /// letter characters are lowercase.+ pub fn is_lowercase(&self) -> bool {+ self.orth_info.contains(OrthFlags::LOWERCASE)+ }+ /// Does the lexeme for this word cover a titlecase variant? (e.g., "Hello")+ ///+ /// This returns true if the word is in titlecase form, which means:+ /// - The first letter is uppercase+ /// - All other letters are lowercase+ /// - The word is at least 2 characters long+ ///+ /// Examples: "Hello", "World"+ ///+ /// Note: Words with internal capital letters (like "McDonald") or apostrophes (like "O'Reilly")+ /// are not considered titlecase - they are classified as UPPER_CAMEL instead.+ pub fn is_titlecase(&self) -> bool {+ self.orth_info.contains(OrthFlags::TITLECASE)+ }+ /// Does the lexeme for this word cover an all-uppercase variant? (e.g., "HELLO")+ ///+ /// This returns true if all letters in the word are uppercase. Words containing+ /// non-letter characters (like numbers or symbols) are only considered if all+ /// letter characters are uppercase.+ ///+ /// Examples: "HELLO", "NASA", "I"+ pub fn is_allcaps(&self) -> bool {+ self.orth_info.contains(OrthFlags::ALLCAPS)+ }+ /// Does the lexeme for this word cover a lower camel case variant? (e.g., "helloWorld")+ ///+ /// This returns true if the word is in lower camel case, which means:+ /// - The first letter is lowercase+ /// - There is at least one uppercase letter after the first character+ /// - The word must be at least 2 characters long+ ///+ /// Examples: "helloWorld", "getHTTPResponse", "eBay"+ ///+ /// Note: Single words that are all lowercase will return false.+ /// Words starting with an uppercase letter will return false (those would be UpperCamel).+ pub fn is_lower_camel(&self) -> bool {+ self.orth_info.contains(OrthFlags::LOWER_CAMEL)+ }+ /// Does the lexeme for this word cover an upper camel case / pascal case variant? (e.g., "HelloWorld")+ ///+ /// This returns true if the word is in upper camel case (also known as Pascal case), which means:+ /// - The first letter is uppercase+ /// - There is at least one other uppercase letter after the first character+ /// - There is at least one lowercase letter after the first uppercase letter+ /// - The word must be at least 3 characters long+ ///+ /// Examples:+ /// - "HelloWorld" (standard Pascal case)+ /// - "McDonald" (name with internal caps)+ /// - "O'Reilly" (name with apostrophe and internal caps)+ /// - "HttpRequest" (initialism followed by word)+ ///+ /// Note: Single words that are titlecase (like "Hello") will return false.+ /// Words that are all uppercase (like "NASA") will also return false.+ pub fn is_upper_camel(&self) -> bool {+ self.orth_info.contains(OrthFlags::UPPER_CAMEL)+ }+/// Same thing as [`Self::or`], except in-place rather than a clone.pub fn append(&mut self, other: &Self) -> &mut Self {*self = self.or(other);@@ -673,8 +749,7 @@ impl VerbData {}// nouns can be both singular and plural: "aircraft", "biceps", "fish", "sheep"-// TODO other noun properties may be worth adding:-// TODO count vs mass; abstract+// TODO other noun properties may be worth adding: abstract#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Hash, Default)]pub struct NounData {pub is_proper: Option<bool>,@@ -994,12 +1069,12 @@ impl Default for DialectFlags {}#[cfg(test)]-mod tests {+pub mod tests {use crate::WordMetadata;use crate::spell::{Dictionary, FstDictionary};// Helper function to get word metadata from the curated dictionary- fn md(word: &str) -> WordMetadata {+ pub fn md(word: &str) -> WordMetadata {FstDictionary::curated().get_word_metadata_str(word).unwrap_or_else(|| panic!("Word '{word}' not found in dictionary"))diff --git a/harper-core/src/word_metadata_orthography.rs b/harper-core/src/word_metadata_orthography.rsnew file mode 100644index 000000000..b20213944--- /dev/null+++ b/harper-core/src/word_metadata_orthography.rs@@ -0,0 +1,102 @@+use serde::{Deserialize, Serialize};++/// Orthography information.+pub enum Orthography {+ /// Every char that is a letter is lowercase.+ Lowercase = 1 << 0,+ /// First char is uppercase, the rest is lowercase (but multi-word?)+ Titlecase = 1 << 1,+ /// Every char that is a letter is uppercase (including single-letter uppercase)+ AllCaps = 1 << 2,+ /// Starts with a lowercase letter but also contains uppercase letters.+ LowerCamel = 1 << 3,+ /// Starts with an uppercase letter but also contains lowercase letters. (Superset of Titlecase.)+ UpperCamel = 1 << 4,+ /// Contains at least one space.+ Multiword = 1 << 5,+ /// Contains at least one hyphen.+ Hyphenated = 1 << 6,+ /// Contains an apostrophe, so it's a possessive or a contraction.+ Apostrophe = 1 << 7,+}++/// The underlying type used for OrthographyFlags.+/// At the time of writing, this is currently a `u8`. If we want to define more than 8 orthographic+/// properties in the future, we will need to switch this to a larger type.+type OrthographyFlagsUnderlyingType = u8;++bitflags::bitflags! {+ /// A collection of bit flags used to represent orthographic properties of a word.+ ///+ /// This is generally used to allow a word (or similar) to be tagged with multiple orthographic+ /// properties.+ #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, PartialOrd, Serialize)]+ pub struct OrthFlags: OrthographyFlagsUnderlyingType {+ const LOWERCASE = Orthography::Lowercase as OrthographyFlagsUnderlyingType;+ const TITLECASE = Orthography::Titlecase as OrthographyFlagsUnderlyingType;+ const ALLCAPS = Orthography::AllCaps as OrthographyFlagsUnderlyingType;+ const LOWER_CAMEL = Orthography::LowerCamel as OrthographyFlagsUnderlyingType;+ const UPPER_CAMEL = Orthography::UpperCamel as OrthographyFlagsUnderlyingType;+ const MULTIWORD = Orthography::Multiword as OrthographyFlagsUnderlyingType;+ const HYPHENATED = Orthography::Hyphenated as OrthographyFlagsUnderlyingType;+ const APOSTROPHE = Orthography::Apostrophe as OrthographyFlagsUnderlyingType;+ }+}+impl Default for OrthFlags {+ fn default() -> Self {+ Self::empty()+ }+}++#[cfg(test)]+mod tests {+ use crate::word_metadata::tests::md;+ use crate::word_metadata_orthography::OrthFlags;++ #[test]+ fn australia_lexeme_is_titlecase_even_when_word_is_lowercase() {+ assert!(md("australia").orth_info.contains(OrthFlags::TITLECASE));+ }++ #[test]+ fn australia_lexeme_is_titlecase_even_when_word_is_all_caps() {+ assert!(md("AUSTRALIA").orth_info.contains(OrthFlags::TITLECASE));+ }++ #[test]+ fn australia_lexeme_is_titlecase_even_when_word_is_mixed_case() {+ assert!(md("AuStrAlIA").orth_info.contains(OrthFlags::TITLECASE));+ }++ #[test]+ fn db_and_kw_symbols_are_lower_camel_case() {+ // dB, kW+ assert!(md("db").orth_info.contains(OrthFlags::LOWER_CAMEL));+ }++ #[test]+ fn am_is_lowercase_and_titlecase_and_all_caps() {+ // am, Am, AM+ let metadata = md("am");+ assert!(metadata.orth_info.contains(OrthFlags::LOWERCASE));+ assert!(metadata.orth_info.contains(OrthFlags::TITLECASE));+ assert!(metadata.orth_info.contains(OrthFlags::ALLCAPS));+ }++ #[test]+ fn reading_is_both_lowercase_and_titlecase() {+ // Reading is a town in England+ let metadata = md("reading");+ assert!(metadata.orth_info.contains(OrthFlags::LOWERCASE));+ assert!(metadata.orth_info.contains(OrthFlags::TITLECASE));+ }++ #[test]+ fn ebay_and_esim_are_lower_camel() {+ // eBay eSIM+ let md1 = md("ebay");+ assert!(md1.orth_info.contains(OrthFlags::LOWER_CAMEL));+ let md2 = md("esim");+ assert!(md2.orth_info.contains(OrthFlags::LOWER_CAMEL));+ }+}