Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(cli): chunk annotations (#2450)
Steve Kurch8dc42c592e4b
diff --git a/harper-cli/src/annotate.rs b/harper-cli/src/annotate.rsnew file mode 100644index 000000000..b174eebca--- /dev/null+++ b/harper-cli/src/annotate.rs@@ -0,0 +1,171 @@+use std::{borrow::Cow, ops::Range};++use ariadne::{Color, Label, Report, ReportKind};+use clap::ValueEnum;+use harper_core::{Document, Span, TokenKind, TokenStringExt};+use strum::IntoEnumIterator;++/// Represents an annotation.+pub(super) struct Annotation {+ /// The range the annotation covers in the source. For instance, this might be a single word.+ span: Span<char>,+ /// The message displayed by the annotation.+ annotation_text: String,+ /// The color of the annotation.+ color: Color,+}+impl Annotation {+ /// Converts the annotation into an [`ariadne::Label`].+ #[must_use]+ pub(super) fn into_label(+ self,+ input_identifier: &str,+ ) -> Label<(&str, std::ops::Range<usize>)> {+ Label::new((input_identifier, self.span.into()))+ .with_message(self.annotation_text)+ .with_color(self.color)+ }++ /// Gets an iterator of annotation `Label` from the given document.+ ///+ /// This is similar to [`Self::iter_from_document`], but this additionally converts+ /// the [`Annotation`] into [`ariadne::Label`] for convenience.+ pub(super) fn iter_labels_from_document<'inpt_id>(+ annotation_type: AnnotationType,+ document: &Document,+ input_identifier: &'inpt_id str,+ ) -> impl Iterator<Item = Label<(&'inpt_id str, std::ops::Range<usize>)>> {+ Self::iter_from_document(annotation_type, document)+ .map(|annotation| annotation.into_label(input_identifier))+ }++ /// Gets an iterator of [`Annotation`] for a given document. The annotations will be based on+ /// `annotation_type`.+ fn iter_from_document(+ annotation_type: AnnotationType,+ document: &Document,+ ) -> Box<dyn Iterator<Item = Self> + '_> {+ match annotation_type {+ AnnotationType::Upos => Box::new({+ document.tokens().filter_map(|token| {+ let span = token.span;+ if let TokenKind::Word(Some(metadata)) = &token.kind {+ // Only annotate words (with dict word metadata) for `AnnotationType::Upos`.+ let pos_tag = metadata.pos_tag;+ Some(Self {+ span,+ annotation_text: pos_tag+ .map_or("NONE".to_owned(), |upos| upos.to_string()),+ color: pos_tag.map_or(Color::Red, get_color_for_enum_variant),+ })+ } else {+ // Not a word, or a word with no metadata.+ None+ }+ })+ }),+ AnnotationType::Chunks => Box::new(+ document+ .iter_chunks()+ .zip(RandomColorIter::new())+ .enumerate()+ .map(|(i, (chunk, color))| Self {+ span: chunk.span().unwrap(),+ annotation_text: i.to_string(),+ color,+ }),+ ),+ }+ }+}++/// Represents how the tokens should be annotated.+#[derive(Debug, Clone, Copy, ValueEnum)]+pub(super) enum AnnotationType {+ /// UPOS (part of speech)+ Upos,+ Chunks,+}+impl AnnotationType {+ /// Build a [`Report`] from the provided [`Document`], `input_identifier`, and `report_title`.+ pub(super) fn build_report<'input_id>(+ &self,+ doc: &Document,+ input_identifier: &'input_id str,+ report_title: &'input_id str,+ ) -> Report<'input_id, (&'input_id str, Range<usize>)> {+ Report::build(+ ReportKind::Custom(report_title, Color::Blue),+ input_identifier,+ 0,+ )+ .with_labels(Annotation::iter_labels_from_document(+ *self,+ doc,+ input_identifier,+ ))+ .finish()+ }++ /// The title that should be used for the printed output.+ pub(super) fn get_title(&self) -> &'static str {+ match self {+ AnnotationType::Upos => "UPOS Tags",+ AnnotationType::Chunks => "Chunks",+ }+ }++ /// The title that should be used for the printed output, with added tags.+ ///+ /// The tags are used to provide additional information about the output.+ pub(super) fn get_title_with_tags(&self, tags: &[&str]) -> Cow<'static, str> {+ if tags.is_empty() {+ self.get_title().into()+ } else {+ let tags = tags.join(", ");+ (self.get_title().to_owned() + &format!(" ({tags})")).into()+ }+ }+}++/// An infinite iterator that produces random colors. This uses a fixed seed, so all instances of+/// this iterator will produce colors in the same order.+struct RandomColorIter {+ color_gen: ariadne::ColorGenerator,+}+impl RandomColorIter {+ fn new() -> Self {+ Self {+ // Using a lower than default `min_brightness` to hopefully create more distinguishable colors.+ color_gen: ariadne::ColorGenerator::from_state([31715, 3528, 21854], 0.2),+ }+ }+}+impl Iterator for RandomColorIter {+ type Item = Color;+ fn next(&mut self) -> Option<Self::Item> {+ Some(self.color_gen.next())+ }+}++/// Gets a random `Color` for an enum variant.+///+/// A given enum variant's color is consistent, meaning it will not change throughout multiple+/// calls of this function or multiple runs of the application.+#[must_use]+fn get_color_for_enum_variant<T: IntoEnumIterator + PartialEq>(variant_to_color: T) -> Color {+ get_color_for_index(+ T::iter()+ .position(|variant| variant == variant_to_color)+ .unwrap(),+ )+}++/// Gets the nth random `Color` for a numeric index.+///+/// A given index's color is consistent, meaning it will not change throughout multiple calls of+/// this function or multiple runs of the application.+#[must_use]+fn get_color_for_index(idx_to_color: usize) -> Color {+ RandomColorIter::new().nth(idx_to_color).unwrap()+}diff --git a/harper-cli/src/annotate_tokens.rs b/harper-cli/src/annotate_tokens.rsdeleted file mode 100644index 3a738611e..000000000--- a/harper-cli/src/annotate_tokens.rs+++ /dev/null@@ -1,98 +0,0 @@-use ariadne::{Color, Label};-use clap::ValueEnum;-use harper_core::{Document, Span, Token, TokenKind};-use strum::IntoEnumIterator;--/// Represents an annotation.-pub(super) struct Annotation {- /// The range the annotation covers in the source. For instance, this might be a single word.- span: Span<char>,- /// The message displayed by the annotation.- annotation_text: String,- /// The color of the annotation.- color: Color,-}-impl Annotation {- /// Converts the annotation into an [`ariadne::Label`].- #[must_use]- pub(super) fn into_label(- self,- input_identifier: &str,- ) -> Label<(&str, std::ops::Range<usize>)> {- Label::new((input_identifier, self.span.into()))- .with_message(self.annotation_text)- .with_color(self.color)- }-- /// Gets an iterator of annotation `Label` from the given document.- ///- /// This is similar to [`Self::iter_from_document`], but this additionally converts- /// the [`Annotation`] into [`ariadne::Label`] for convenience.- pub(super) fn iter_labels_from_document<'inpt_id>(- annotation_type: AnnotationType,- document: &Document,- input_identifier: &'inpt_id str,- ) -> impl Iterator<Item = Label<(&'inpt_id str, std::ops::Range<usize>)>> {- Self::iter_from_document(annotation_type, document)- .map(|annotation| annotation.into_label(input_identifier))- }-- /// Constructs an [`Annotation`] from the given `token` based on the given `annotation_type`.- ///- /// This will return `None` when the `token` should not be annotated according to- /// `annotation_type`.- #[must_use]- fn from_token(annotation_type: AnnotationType, token: &Token) -> Option<Self> {- let span = token.span;- match annotation_type {- AnnotationType::Upos => {- if let TokenKind::Word(Some(metadata)) = &token.kind {- // Only annotate words (with dict word metadata) for `AnnotationType::Upos`.- let pos_tag = metadata.pos_tag;- Some(Self {- span,- annotation_text: pos_tag.map_or("NONE".to_owned(), |upos| upos.to_string()),- color: pos_tag.map_or(Color::Red, get_color_for_enum_variant),- })- } else {- // Not a word, or a word with no metadata.- None- }- }- }- }-- /// Gets an iterator of [`Annotation`] for a given document. The annotations will be based on- /// `annotation_type`.- fn iter_from_document(- annotation_type: AnnotationType,- document: &Document,- ) -> impl Iterator<Item = Self> {- document- .tokens()- .filter_map(move |token| Self::from_token(annotation_type, token))- }-}--/// Represents how the tokens should be annotated.-#[derive(Debug, Clone, Copy, ValueEnum)]-pub(super) enum AnnotationType {- /// UPOS (part of speech)- Upos,-}--/// Gets a random `Color` for an enum variant.-///-/// A given enum variant's color is consistent, meaning it will not change throughout multiple-/// calls of this function or multiple runs of the application.-#[must_use]-fn get_color_for_enum_variant<T: IntoEnumIterator + PartialEq>(variant_to_color: T) -> Color {- // Using a lower than default `min_brightness` to hopefully create more distinguishable colors.- let mut color_gen = ariadne::ColorGenerator::from_state([31715, 3528, 21854], 0.2);- T::iter()- // Note: `ColorGenerator` does not implement `Iterator`, so we can't just zip it.- .map(|enum_variant| (enum_variant, color_gen.next()))- .find(|(enum_variant, _)| *enum_variant == variant_to_color)- .unwrap()- .1-}diff --git a/harper-cli/src/input/single_input.rs b/harper-cli/src/input/single_input.rsindex db01694d6..b42cfab7c 100644--- a/harper-cli/src/input/single_input.rs+++ b/harper-cli/src/input/single_input.rs@@ -28,9 +28,19 @@ pub(crate) trait SingleInputTrait: InputTrait {&self,markdown_options: MarkdownOptions,dictionary: &dyn Dictionary,+ ) -> anyhow::Result<(Document, Cow<'_, str>)> {+ self.load_with_parser(&self.get_parser(markdown_options), dictionary)+ }++ /// Loads the contained file/string into a conventional format using the provided+ /// parser. Returns a `Result` containing a tuple of a `Document` and its corresponding source+ /// text as a string.+ fn load_with_parser(+ &self,+ parser: &dyn Parser,+ dictionary: &dyn Dictionary,) -> anyhow::Result<(Document, Cow<'_, str>)> {let text = self.get_content()?;- let parser = self.get_parser(markdown_options);Ok((Document::new(&text, &parser, &dictionary), text))}diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex 6d55e5ade..68cd2ce9d 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -14,7 +14,7 @@ use ariadne::{Color, Label, Report, ReportKind, Source};use clap::Parser;use dirs::{config_dir, data_local_dir};use harper_core::linting::LintGroup;-use harper_core::parsers::MarkdownOptions;+use harper_core::parsers::{IsolateEnglish, MarkdownOptions};use harper_core::weir::WeirLinter;use harper_core::{CharStringExt, Dialect, DictWordMetadata, OrthFlags, Span, TokenKind, TokenStringExt,@@ -32,8 +32,8 @@ use input::{single_input::{SingleInput, SingleInputOptionExt, SingleInputTrait},};-mod annotate_tokens;-use annotate_tokens::{Annotation, AnnotationType};+mod annotate;+use annotate::AnnotationType;mod lint;use crate::lint::lint;@@ -88,14 +88,17 @@ enum Args {#[arg(short, long)]include_newlines: bool,},- /// Parse a provided document and annotate its tokens.- AnnotateTokens {+ /// Parse and annotate a provided document.+ Annotate {/// The text or file you wish to parse. If not provided, it will be read from standard/// input.input: Option<SingleInput>,- /// How the tokens should be annotated.+ /// How the document should be annotated.#[arg(short, long, value_enum, default_value_t = AnnotationType::Upos)]annotation_type: AnnotationType,+ /// Attempt to detect and ignore non-English spans of text.+ #[arg(short, long)]+ isolate_english: bool,},/// Get the metadata associated with one or more words.Metadata {@@ -292,32 +295,39 @@ fn main() -> anyhow::Result<()> {Ok(())}- Args::AnnotateTokens {+ Args::Annotate {input,annotation_type,+ isolate_english,} => {// Try to read from standard input if `input` was not provided.let input = input.unwrap_or_read_from_stdin();+ let parser = if isolate_english {+ Box::new(IsolateEnglish::new(+ input.get_parser(markdown_options),+ &curated_dictionary,+ ))+ } else {+ input.get_parser(markdown_options)+ };+// Load the file/text.- let (doc, source) = input.load(markdown_options, &curated_dictionary)?;+ let (doc, source) = input.load_with_parser(&parser, &curated_dictionary)?;let input_identifier = input.get_identifier();- let mut report_builder = Report::build(- ReportKind::Custom("AnnotateTokens", Color::Blue),- &*input_identifier,- 0,- );-- report_builder = report_builder.with_labels(Annotation::iter_labels_from_document(- annotation_type,- &doc,- &input_identifier,- ));-- let report = report_builder.finish();- report.print((&*input_identifier, Source::from(source)))?;+ annotation_type+ .build_report(+ &doc,+ &input_identifier,+ &annotation_type.get_title_with_tags(if isolate_english {+ &["Isolate english"]+ } else {+ &[]+ }),+ )+ .print((&*input_identifier, Source::from(source)))?;Ok(())}diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex e316c09c9..7e0a96bcc 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -19,8 +19,8 @@ pub use plain_english::PlainEnglish;use crate::{LSend, Token, TokenStringExt};-#[cfg_attr(feature = "concurrent", blanket(derive(Box, Arc)))]-#[cfg_attr(not(feature = "concurrent"), blanket(derive(Box, Rc)))]+#[cfg_attr(feature = "concurrent", blanket(derive(Ref, Box, Arc)))]+#[cfg_attr(not(feature = "concurrent"), blanket(derive(Ref, Box, Rc)))]pub trait Parser: LSend {fn parse(&self, source: &[char]) -> Vec<Token>;}