Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(cli): add `annotate-tokens` subcommand (#1533)

Steve Kurchb325d5dbe24f
diff --git a/Cargo.lock b/Cargo.lock
index b775bd68a..6ec3c36e5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -840,6 +840,8 @@ dependencies = [
"hashbrown 0.15.4",
"serde",
"serde_json",
+ "strum",
+ "strum_macros",
]
[[package]]
diff --git a/harper-cli/Cargo.toml b/harper-cli/Cargo.toml
index 76b266cff..489df4b54 100644
--- a/harper-cli/Cargo.toml
+++ b/harper-cli/Cargo.toml
@@ -19,6 +19,8 @@ harper-typst = { path = "../harper-typst", version = "0.49.0" }
hashbrown = "0.15.4"
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
+strum = "0.27.1"
+strum_macros = "0.27.1"
[features]
default = []
diff --git a/harper-cli/src/annotate_tokens.rs b/harper-cli/src/annotate_tokens.rs
new file mode 100644
index 000000000..f01fb2cf2
--- /dev/null
+++ b/harper-cli/src/annotate_tokens.rs
@@ -0,0 +1,98 @@
+use ariadne::{Color, Label};
+use clap::ValueEnum;
+use harper_core::{Document, Span, Token, TokenKind};
+use strum::IntoEnumIterator;
+
+/// Represents an annotation.
+pub(super) struct Annotation {
+ /// The range the annotation covers in the source. For instance, this might be a single word.
+ span: Span,
+ /// The message displayed by the annotation.
+ annotation_text: String,
+ /// The color of the annotation.
+ color: Color,
+}
+impl Annotation {
+ /// Converts the annotation into an [`ariadne::Label`].
+ #[must_use]
+ pub(super) fn into_label(
+ self,
+ input_identifier: &str,
+ ) -> Label<(&str, std::ops::Range<usize>)> {
+ Label::new((input_identifier, self.span.into()))
+ .with_message(self.annotation_text)
+ .with_color(self.color)
+ }
+
+ /// Gets an iterator of annotation `Label` from the given document.
+ ///
+ /// This is similar to [`self::iter_from_document()`], but this additionally converts
+ /// the [`Annotation`] into [`ariadne::Label`] for convenience.
+ pub(super) fn iter_labels_from_document<'inpt_id>(
+ annotation_type: AnnotationType,
+ document: &Document,
+ input_identifier: &'inpt_id str,
+ ) -> impl Iterator<Item = Label<(&'inpt_id str, std::ops::Range<usize>)>> {
+ Self::iter_from_document(annotation_type, document)
+ .map(|annotation| annotation.into_label(input_identifier))
+ }
+
+ /// Constructs an [`Annotation`] from the given `token` based on the given `annotation_type`.
+ ///
+ /// This will return `None` when the `token` should not be annotated according to
+ /// `annotation_type`.
+ #[must_use]
+ fn from_token(annotation_type: AnnotationType, token: &Token) -> Option<Self> {
+ let span = token.span;
+ match annotation_type {
+ AnnotationType::Upos => {
+ if let TokenKind::Word(Some(metadata)) = &token.kind {
+ // Only annotate words (with word metadata) for `AnnotationType::Upos`.
+ let pos_tag = metadata.pos_tag;
+ Some(Self {
+ span,
+ annotation_text: pos_tag.map_or("NONE".to_owned(), |upos| upos.to_string()),
+ color: pos_tag.map_or(Color::Red, get_color_for_enum_variant),
+ })
+ } else {
+ // Not a word, or a word with no metadata.
+ None
+ }
+ }
+ }
+ }
+
+ /// Gets an iterator of [`Annotation`] for a given document. The annotations will be based on
+ /// `annotation_type`.
+ fn iter_from_document(
+ annotation_type: AnnotationType,
+ document: &Document,
+ ) -> impl Iterator<Item = Self> {
+ document
+ .tokens()
+ .filter_map(move |token| Self::from_token(annotation_type, token))
+ }
+}
+
+/// Represents how the tokens should be annotated.
+#[derive(Debug, Clone, Copy, ValueEnum)]
+pub(super) enum AnnotationType {
+ /// UPOS (part of speech)
+ Upos,
+}
+
+/// Gets a random `Color` for an enum variant.
+///
+/// A given enum variant's color is consistent, meaning it will not change throughout multiple
+/// calls of this function or multiple runs of the application.
+#[must_use]
+fn get_color_for_enum_variant<T: IntoEnumIterator + PartialEq>(variant_to_color: T) -> Color {
+ // Using a lower than default `min_brightness` to hopefully create more distinguishable colors.
+ let mut color_gen = ariadne::ColorGenerator::from_state([31715, 3528, 21854], 0.2);
+ T::iter()
+ // Note: `ColorGenerator` does not implement `Iterator`, so we can't just zip it.
+ .map(|enum_variant| (enum_variant, color_gen.next()))
+ .find(|(enum_variant, _)| *enum_variant == variant_to_color)
+ .unwrap()
+ .1
+}
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index 9489f405f..5a7db855b 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -27,6 +27,9 @@ use serde::Serialize;
mod input;
use input::Input;
+mod annotate_tokens;
+use annotate_tokens::{Annotation, AnnotationType};
+
/// A debugging tool for the Harper grammar checker.
#[derive(Debug, Parser)]
#[command(version, about)]
@@ -69,6 +72,15 @@ enum Args {
#[arg(short, long)]
include_newlines: bool,
},
+ /// Parse a provided document and annotate its tokens.
+ AnnotateTokens {
+ /// The text or file you wish to parse. If not provided, it will be read from standard
+ /// input.
+ input: Option<Input>,
+ /// How the tokens should be annotated.
+ #[arg(short, long, value_enum, default_value_t = AnnotationType::Upos)]
+ annotation_type: AnnotationType,
+ },
/// Get the metadata associated with a particular word.
Metadata { word: String },
/// Get all the forms of a word using the affixes.
@@ -268,6 +280,35 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
+ Args::AnnotateTokens {
+ input,
+ annotation_type,
+ } => {
+ // Try to read from standard input if `input` was not provided.
+ let input = input.unwrap_or_else(|| Input::try_from_stdin().unwrap());
+
+ // Load the file/text.
+ let (doc, source) = input.load(markdown_options, &dictionary)?;
+
+ let input_identifier = input.get_identifier();
+
+ let mut report_builder = Report::build(
+ ReportKind::Custom("AnnotateTokens", Color::Blue),
+ &*input_identifier,
+ 0,
+ );
+
+ report_builder = report_builder.with_labels(Annotation::iter_labels_from_document(
+ annotation_type,
+ &doc,
+ &input_identifier,
+ ));
+
+ let report = report_builder.finish();
+ report.print((&*input_identifier, Source::from(source)))?;
+
+ Ok(())
+ }
Args::Words => {
let mut word_str = String::new();
diff --git a/harper-pos-utils/src/upos.rs b/harper-pos-utils/src/upos.rs
index 5358cae81..2d5f43a2d 100644
--- a/harper-pos-utils/src/upos.rs
+++ b/harper-pos-utils/src/upos.rs
@@ -1,3 +1,5 @@
+use std::fmt::Display;
+
use is_macro::Is;
use serde::{Deserialize, Serialize};
use strum_macros::{AsRefStr, EnumIter};
@@ -82,3 +84,27 @@ impl UPOS {
matches!(self, Self::NOUN | Self::PROPN | Self::PRON)
}
}
+
+impl Display for UPOS {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let desc = match self {
+ UPOS::ADJ => "Adjective",
+ UPOS::ADP => "Adposition",
+ UPOS::ADV => "Adverb",
+ UPOS::AUX => "Auxiliary",
+ UPOS::CCONJ => "Coordinating conjunction",
+ UPOS::DET => "Determiner",
+ UPOS::INTJ => "Interjection",
+ UPOS::NOUN => "Noun",
+ UPOS::NUM => "Numeral",
+ UPOS::PART => "Particle",
+ UPOS::PRON => "Pronoun",
+ UPOS::PROPN => "Proper noun",
+ UPOS::PUNCT => "Punctuation",
+ UPOS::SCONJ => "Subordinating conjunction",
+ UPOS::SYM => "Symbol",
+ UPOS::VERB => "Verb",
+ };
+ write!(f, "{desc}")
+ }
+}