Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(cli): add `annotate-tokens` subcommand (#1533)
Steve Kurchb325d5dbe24f
diff --git a/Cargo.lock b/Cargo.lockindex b775bd68a..6ec3c36e5 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -840,6 +840,8 @@ dependencies = ["hashbrown 0.15.4","serde","serde_json",+ "strum",+ "strum_macros",][[package]]diff --git a/harper-cli/Cargo.toml b/harper-cli/Cargo.tomlindex 76b266cff..489df4b54 100644--- a/harper-cli/Cargo.toml+++ b/harper-cli/Cargo.toml@@ -19,6 +19,8 @@ harper-typst = { path = "../harper-typst", version = "0.49.0" }hashbrown = "0.15.4"serde = { version = "1.0.219", features = ["derive"] }serde_json = "1.0.140"+strum = "0.27.1"+strum_macros = "0.27.1"[features]default = []diff --git a/harper-cli/src/annotate_tokens.rs b/harper-cli/src/annotate_tokens.rsnew file mode 100644index 000000000..f01fb2cf2--- /dev/null+++ b/harper-cli/src/annotate_tokens.rs@@ -0,0 +1,98 @@+use ariadne::{Color, Label};+use clap::ValueEnum;+use harper_core::{Document, Span, Token, TokenKind};+use strum::IntoEnumIterator;++/// Represents an annotation.+pub(super) struct Annotation {+ /// The range the annotation covers in the source. For instance, this might be a single word.+ span: Span,+ /// The message displayed by the annotation.+ annotation_text: String,+ /// The color of the annotation.+ color: Color,+}+impl Annotation {+ /// Converts the annotation into an [`ariadne::Label`].+ #[must_use]+ pub(super) fn into_label(+ self,+ input_identifier: &str,+ ) -> Label<(&str, std::ops::Range<usize>)> {+ Label::new((input_identifier, self.span.into()))+ .with_message(self.annotation_text)+ .with_color(self.color)+ }++ /// Gets an iterator of annotation `Label` from the given document.+ ///+ /// This is similar to [`self::iter_from_document()`], but this additionally converts+ /// the [`Annotation`] into [`ariadne::Label`] for convenience.+ pub(super) fn iter_labels_from_document<'inpt_id>(+ annotation_type: AnnotationType,+ document: &Document,+ input_identifier: &'inpt_id str,+ ) -> impl Iterator<Item = Label<(&'inpt_id str, std::ops::Range<usize>)>> {+ Self::iter_from_document(annotation_type, document)+ .map(|annotation| annotation.into_label(input_identifier))+ }++ /// Constructs an [`Annotation`] from the given `token` based on the given `annotation_type`.+ ///+ /// This will return `None` when the `token` should not be annotated according to+ /// `annotation_type`.+ #[must_use]+ fn from_token(annotation_type: AnnotationType, token: &Token) -> Option<Self> {+ let span = token.span;+ match annotation_type {+ AnnotationType::Upos => {+ if let TokenKind::Word(Some(metadata)) = &token.kind {+ // Only annotate words (with word metadata) for `AnnotationType::Upos`.+ let pos_tag = metadata.pos_tag;+ Some(Self {+ span,+ annotation_text: pos_tag.map_or("NONE".to_owned(), |upos| upos.to_string()),+ color: pos_tag.map_or(Color::Red, get_color_for_enum_variant),+ })+ } else {+ // Not a word, or a word with no metadata.+ None+ }+ }+ }+ }++ /// Gets an iterator of [`Annotation`] for a given document. The annotations will be based on+ /// `annotation_type`.+ fn iter_from_document(+ annotation_type: AnnotationType,+ document: &Document,+ ) -> impl Iterator<Item = Self> {+ document+ .tokens()+ .filter_map(move |token| Self::from_token(annotation_type, token))+ }+}++/// Represents how the tokens should be annotated.+#[derive(Debug, Clone, Copy, ValueEnum)]+pub(super) enum AnnotationType {+ /// UPOS (part of speech)+ Upos,+}++/// Gets a random `Color` for an enum variant.+///+/// A given enum variant's color is consistent, meaning it will not change throughout multiple+/// calls of this function or multiple runs of the application.+#[must_use]+fn get_color_for_enum_variant<T: IntoEnumIterator + PartialEq>(variant_to_color: T) -> Color {+ // Using a lower than default `min_brightness` to hopefully create more distinguishable colors.+ let mut color_gen = ariadne::ColorGenerator::from_state([31715, 3528, 21854], 0.2);+ T::iter()+ // Note: `ColorGenerator` does not implement `Iterator`, so we can't just zip it.+ .map(|enum_variant| (enum_variant, color_gen.next()))+ .find(|(enum_variant, _)| *enum_variant == variant_to_color)+ .unwrap()+ .1+}diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex 9489f405f..5a7db855b 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -27,6 +27,9 @@ use serde::Serialize;mod input;use input::Input;+mod annotate_tokens;+use annotate_tokens::{Annotation, AnnotationType};+/// A debugging tool for the Harper grammar checker.#[derive(Debug, Parser)]#[command(version, about)]@@ -69,6 +72,15 @@ enum Args {#[arg(short, long)]include_newlines: bool,},+ /// Parse a provided document and annotate its tokens.+ AnnotateTokens {+ /// The text or file you wish to parse. If not provided, it will be read from standard+ /// input.+ input: Option<Input>,+ /// How the tokens should be annotated.+ #[arg(short, long, value_enum, default_value_t = AnnotationType::Upos)]+ annotation_type: AnnotationType,+ },/// Get the metadata associated with a particular word.Metadata { word: String },/// Get all the forms of a word using the affixes.@@ -268,6 +280,35 @@ fn main() -> anyhow::Result<()> {Ok(())}+ Args::AnnotateTokens {+ input,+ annotation_type,+ } => {+ // Try to read from standard input if `input` was not provided.+ let input = input.unwrap_or_else(|| Input::try_from_stdin().unwrap());++ // Load the file/text.+ let (doc, source) = input.load(markdown_options, &dictionary)?;++ let input_identifier = input.get_identifier();++ let mut report_builder = Report::build(+ ReportKind::Custom("AnnotateTokens", Color::Blue),+ &*input_identifier,+ 0,+ );++ report_builder = report_builder.with_labels(Annotation::iter_labels_from_document(+ annotation_type,+ &doc,+ &input_identifier,+ ));++ let report = report_builder.finish();+ report.print((&*input_identifier, Source::from(source)))?;++ Ok(())+ }Args::Words => {let mut word_str = String::new();diff --git a/harper-pos-utils/src/upos.rs b/harper-pos-utils/src/upos.rsindex 5358cae81..2d5f43a2d 100644--- a/harper-pos-utils/src/upos.rs+++ b/harper-pos-utils/src/upos.rs@@ -1,3 +1,5 @@+use std::fmt::Display;+use is_macro::Is;use serde::{Deserialize, Serialize};use strum_macros::{AsRefStr, EnumIter};@@ -82,3 +84,27 @@ impl UPOS {matches!(self, Self::NOUN | Self::PROPN | Self::PRON)}}++impl Display for UPOS {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ let desc = match self {+ UPOS::ADJ => "Adjective",+ UPOS::ADP => "Adposition",+ UPOS::ADV => "Adverb",+ UPOS::AUX => "Auxiliary",+ UPOS::CCONJ => "Coordinating conjunction",+ UPOS::DET => "Determiner",+ UPOS::INTJ => "Interjection",+ UPOS::NOUN => "Noun",+ UPOS::NUM => "Numeral",+ UPOS::PART => "Particle",+ UPOS::PRON => "Pronoun",+ UPOS::PROPN => "Proper noun",+ UPOS::PUNCT => "Punctuation",+ UPOS::SCONJ => "Subordinating conjunction",+ UPOS::SYM => "Symbol",+ UPOS::VERB => "Verb",+ };+ write!(f, "{desc}")+ }+}