Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(cli): chunk annotations (#2450)

Steve Kurch8dc42c592e4b
diff --git a/harper-cli/src/annotate.rs b/harper-cli/src/annotate.rs
new file mode 100644
index 000000000..b174eebca
--- /dev/null
+++ b/harper-cli/src/annotate.rs
@@ -0,0 +1,171 @@
+use std::{borrow::Cow, ops::Range};
+
+use ariadne::{Color, Label, Report, ReportKind};
+use clap::ValueEnum;
+use harper_core::{Document, Span, TokenKind, TokenStringExt};
+use strum::IntoEnumIterator;
+
+/// Represents an annotation.
+pub(super) struct Annotation {
+ /// The range the annotation covers in the source. For instance, this might be a single word.
+ span: Span<char>,
+ /// The message displayed by the annotation.
+ annotation_text: String,
+ /// The color of the annotation.
+ color: Color,
+}
+impl Annotation {
+ /// Converts the annotation into an [`ariadne::Label`].
+ #[must_use]
+ pub(super) fn into_label(
+ self,
+ input_identifier: &str,
+ ) -> Label<(&str, std::ops::Range<usize>)> {
+ Label::new((input_identifier, self.span.into()))
+ .with_message(self.annotation_text)
+ .with_color(self.color)
+ }
+
+ /// Gets an iterator of annotation `Label` from the given document.
+ ///
+ /// This is similar to [`Self::iter_from_document`], but this additionally converts
+ /// the [`Annotation`] into [`ariadne::Label`] for convenience.
+ pub(super) fn iter_labels_from_document<'inpt_id>(
+ annotation_type: AnnotationType,
+ document: &Document,
+ input_identifier: &'inpt_id str,
+ ) -> impl Iterator<Item = Label<(&'inpt_id str, std::ops::Range<usize>)>> {
+ Self::iter_from_document(annotation_type, document)
+ .map(|annotation| annotation.into_label(input_identifier))
+ }
+
+ /// Gets an iterator of [`Annotation`] for a given document. The annotations will be based on
+ /// `annotation_type`.
+ fn iter_from_document(
+ annotation_type: AnnotationType,
+ document: &Document,
+ ) -> Box<dyn Iterator<Item = Self> + '_> {
+ match annotation_type {
+ AnnotationType::Upos => Box::new({
+ document.tokens().filter_map(|token| {
+ let span = token.span;
+ if let TokenKind::Word(Some(metadata)) = &token.kind {
+ // Only annotate words (with dict word metadata) for `AnnotationType::Upos`.
+ let pos_tag = metadata.pos_tag;
+ Some(Self {
+ span,
+ annotation_text: pos_tag
+ .map_or("NONE".to_owned(), |upos| upos.to_string()),
+ color: pos_tag.map_or(Color::Red, get_color_for_enum_variant),
+ })
+ } else {
+ // Not a word, or a word with no metadata.
+ None
+ }
+ })
+ }),
+ AnnotationType::Chunks => Box::new(
+ document
+ .iter_chunks()
+ .zip(RandomColorIter::new())
+ .enumerate()
+ .map(|(i, (chunk, color))| Self {
+ span: chunk.span().unwrap(),
+ annotation_text: i.to_string(),
+ color,
+ }),
+ ),
+ }
+ }
+}
+
+/// Represents how the tokens should be annotated.
+#[derive(Debug, Clone, Copy, ValueEnum)]
+pub(super) enum AnnotationType {
+ /// UPOS (part of speech)
+ Upos,
+ Chunks,
+}
+impl AnnotationType {
+ /// Build a [`Report`] from the provided [`Document`], `input_identifier`, and `report_title`.
+ pub(super) fn build_report<'input_id>(
+ &self,
+ doc: &Document,
+ input_identifier: &'input_id str,
+ report_title: &'input_id str,
+ ) -> Report<'input_id, (&'input_id str, Range<usize>)> {
+ Report::build(
+ ReportKind::Custom(report_title, Color::Blue),
+ input_identifier,
+ 0,
+ )
+ .with_labels(Annotation::iter_labels_from_document(
+ *self,
+ doc,
+ input_identifier,
+ ))
+ .finish()
+ }
+
+ /// The title that should be used for the printed output.
+ pub(super) fn get_title(&self) -> &'static str {
+ match self {
+ AnnotationType::Upos => "UPOS Tags",
+ AnnotationType::Chunks => "Chunks",
+ }
+ }
+
+ /// The title that should be used for the printed output, with added tags.
+ ///
+ /// The tags are used to provide additional information about the output.
+ pub(super) fn get_title_with_tags(&self, tags: &[&str]) -> Cow<'static, str> {
+ if tags.is_empty() {
+ self.get_title().into()
+ } else {
+ let tags = tags.join(", ");
+ (self.get_title().to_owned() + &format!(" ({tags})")).into()
+ }
+ }
+}
+
+/// An infinite iterator that produces random colors. This uses a fixed seed, so all instances of
+/// this iterator will produce colors in the same order.
+struct RandomColorIter {
+ color_gen: ariadne::ColorGenerator,
+}
+impl RandomColorIter {
+ fn new() -> Self {
+ Self {
+ // Using a lower than default `min_brightness` to hopefully create more distinguishable colors.
+ color_gen: ariadne::ColorGenerator::from_state([31715, 3528, 21854], 0.2),
+ }
+ }
+}
+impl Iterator for RandomColorIter {
+ type Item = Color;
+ fn next(&mut self) -> Option<Self::Item> {
+ Some(self.color_gen.next())
+ }
+}
+
+/// Gets a random `Color` for an enum variant.
+///
+/// A given enum variant's color is consistent, meaning it will not change throughout multiple
+/// calls of this function or multiple runs of the application.
+#[must_use]
+fn get_color_for_enum_variant<T: IntoEnumIterator + PartialEq>(variant_to_color: T) -> Color {
+ get_color_for_index(
+ T::iter()
+ .position(|variant| variant == variant_to_color)
+ .unwrap(),
+ )
+}
+
+/// Gets the nth random `Color` for a numeric index.
+///
+/// A given index's color is consistent, meaning it will not change throughout multiple calls of
+/// this function or multiple runs of the application.
+#[must_use]
+fn get_color_for_index(idx_to_color: usize) -> Color {
+ RandomColorIter::new().nth(idx_to_color).unwrap()
+}
diff --git a/harper-cli/src/annotate_tokens.rs b/harper-cli/src/annotate_tokens.rs
deleted file mode 100644
index 3a738611e..000000000
--- a/harper-cli/src/annotate_tokens.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-use ariadne::{Color, Label};
-use clap::ValueEnum;
-use harper_core::{Document, Span, Token, TokenKind};
-use strum::IntoEnumIterator;
-
-/// Represents an annotation.
-pub(super) struct Annotation {
- /// The range the annotation covers in the source. For instance, this might be a single word.
- span: Span<char>,
- /// The message displayed by the annotation.
- annotation_text: String,
- /// The color of the annotation.
- color: Color,
-}
-impl Annotation {
- /// Converts the annotation into an [`ariadne::Label`].
- #[must_use]
- pub(super) fn into_label(
- self,
- input_identifier: &str,
- ) -> Label<(&str, std::ops::Range<usize>)> {
- Label::new((input_identifier, self.span.into()))
- .with_message(self.annotation_text)
- .with_color(self.color)
- }
-
- /// Gets an iterator of annotation `Label` from the given document.
- ///
- /// This is similar to [`Self::iter_from_document`], but this additionally converts
- /// the [`Annotation`] into [`ariadne::Label`] for convenience.
- pub(super) fn iter_labels_from_document<'inpt_id>(
- annotation_type: AnnotationType,
- document: &Document,
- input_identifier: &'inpt_id str,
- ) -> impl Iterator<Item = Label<(&'inpt_id str, std::ops::Range<usize>)>> {
- Self::iter_from_document(annotation_type, document)
- .map(|annotation| annotation.into_label(input_identifier))
- }
-
- /// Constructs an [`Annotation`] from the given `token` based on the given `annotation_type`.
- ///
- /// This will return `None` when the `token` should not be annotated according to
- /// `annotation_type`.
- #[must_use]
- fn from_token(annotation_type: AnnotationType, token: &Token) -> Option<Self> {
- let span = token.span;
- match annotation_type {
- AnnotationType::Upos => {
- if let TokenKind::Word(Some(metadata)) = &token.kind {
- // Only annotate words (with dict word metadata) for `AnnotationType::Upos`.
- let pos_tag = metadata.pos_tag;
- Some(Self {
- span,
- annotation_text: pos_tag.map_or("NONE".to_owned(), |upos| upos.to_string()),
- color: pos_tag.map_or(Color::Red, get_color_for_enum_variant),
- })
- } else {
- // Not a word, or a word with no metadata.
- None
- }
- }
- }
- }
-
- /// Gets an iterator of [`Annotation`] for a given document. The annotations will be based on
- /// `annotation_type`.
- fn iter_from_document(
- annotation_type: AnnotationType,
- document: &Document,
- ) -> impl Iterator<Item = Self> {
- document
- .tokens()
- .filter_map(move |token| Self::from_token(annotation_type, token))
- }
-}
-
-/// Represents how the tokens should be annotated.
-#[derive(Debug, Clone, Copy, ValueEnum)]
-pub(super) enum AnnotationType {
- /// UPOS (part of speech)
- Upos,
-}
-
-/// Gets a random `Color` for an enum variant.
-///
-/// A given enum variant's color is consistent, meaning it will not change throughout multiple
-/// calls of this function or multiple runs of the application.
-#[must_use]
-fn get_color_for_enum_variant<T: IntoEnumIterator + PartialEq>(variant_to_color: T) -> Color {
- // Using a lower than default `min_brightness` to hopefully create more distinguishable colors.
- let mut color_gen = ariadne::ColorGenerator::from_state([31715, 3528, 21854], 0.2);
- T::iter()
- // Note: `ColorGenerator` does not implement `Iterator`, so we can't just zip it.
- .map(|enum_variant| (enum_variant, color_gen.next()))
- .find(|(enum_variant, _)| *enum_variant == variant_to_color)
- .unwrap()
- .1
-}
diff --git a/harper-cli/src/input/single_input.rs b/harper-cli/src/input/single_input.rs
index db01694d6..b42cfab7c 100644
--- a/harper-cli/src/input/single_input.rs
+++ b/harper-cli/src/input/single_input.rs
@@ -28,9 +28,19 @@ pub(crate) trait SingleInputTrait: InputTrait {
&self,
markdown_options: MarkdownOptions,
dictionary: &dyn Dictionary,
+ ) -> anyhow::Result<(Document, Cow<'_, str>)> {
+ self.load_with_parser(&self.get_parser(markdown_options), dictionary)
+ }
+
+ /// Loads the contained file/string into a conventional format using the provided
+ /// parser. Returns a `Result` containing a tuple of a `Document` and its corresponding source
+ /// text as a string.
+ fn load_with_parser(
+ &self,
+ parser: &dyn Parser,
+ dictionary: &dyn Dictionary,
) -> anyhow::Result<(Document, Cow<'_, str>)> {
let text = self.get_content()?;
- let parser = self.get_parser(markdown_options);
Ok((Document::new(&text, &parser, &dictionary), text))
}
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index 6d55e5ade..68cd2ce9d 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -14,7 +14,7 @@ use ariadne::{Color, Label, Report, ReportKind, Source};
use clap::Parser;
use dirs::{config_dir, data_local_dir};
use harper_core::linting::LintGroup;
-use harper_core::parsers::MarkdownOptions;
+use harper_core::parsers::{IsolateEnglish, MarkdownOptions};
use harper_core::weir::WeirLinter;
use harper_core::{
CharStringExt, Dialect, DictWordMetadata, OrthFlags, Span, TokenKind, TokenStringExt,
@@ -32,8 +32,8 @@ use input::{
single_input::{SingleInput, SingleInputOptionExt, SingleInputTrait},
};
-mod annotate_tokens;
-use annotate_tokens::{Annotation, AnnotationType};
+mod annotate;
+use annotate::AnnotationType;
mod lint;
use crate::lint::lint;
@@ -88,14 +88,17 @@ enum Args {
#[arg(short, long)]
include_newlines: bool,
},
- /// Parse a provided document and annotate its tokens.
- AnnotateTokens {
+ /// Parse and annotate a provided document.
+ Annotate {
/// The text or file you wish to parse. If not provided, it will be read from standard
/// input.
input: Option<SingleInput>,
- /// How the tokens should be annotated.
+ /// How the document should be annotated.
#[arg(short, long, value_enum, default_value_t = AnnotationType::Upos)]
annotation_type: AnnotationType,
+ /// Attempt to detect and ignore non-English spans of text.
+ #[arg(short, long)]
+ isolate_english: bool,
},
/// Get the metadata associated with one or more words.
Metadata {
@@ -292,32 +295,39 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
- Args::AnnotateTokens {
+ Args::Annotate {
input,
annotation_type,
+ isolate_english,
} => {
// Try to read from standard input if `input` was not provided.
let input = input.unwrap_or_read_from_stdin();
+ let parser = if isolate_english {
+ Box::new(IsolateEnglish::new(
+ input.get_parser(markdown_options),
+ &curated_dictionary,
+ ))
+ } else {
+ input.get_parser(markdown_options)
+ };
+
// Load the file/text.
- let (doc, source) = input.load(markdown_options, &curated_dictionary)?;
+ let (doc, source) = input.load_with_parser(&parser, &curated_dictionary)?;
let input_identifier = input.get_identifier();
- let mut report_builder = Report::build(
- ReportKind::Custom("AnnotateTokens", Color::Blue),
- &*input_identifier,
- 0,
- );
-
- report_builder = report_builder.with_labels(Annotation::iter_labels_from_document(
- annotation_type,
- &doc,
- &input_identifier,
- ));
-
- let report = report_builder.finish();
- report.print((&*input_identifier, Source::from(source)))?;
+ annotation_type
+ .build_report(
+ &doc,
+ &input_identifier,
+ &annotation_type.get_title_with_tags(if isolate_english {
+ &["Isolate english"]
+ } else {
+ &[]
+ }),
+ )
+ .print((&*input_identifier, Source::from(source)))?;
Ok(())
}
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
index e316c09c9..7e0a96bcc 100644
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@ -19,8 +19,8 @@ pub use plain_english::PlainEnglish;
use crate::{LSend, Token, TokenStringExt};
-#[cfg_attr(feature = "concurrent", blanket(derive(Box, Arc)))]
-#[cfg_attr(not(feature = "concurrent"), blanket(derive(Box, Rc)))]
+#[cfg_attr(feature = "concurrent", blanket(derive(Ref, Box, Arc)))]
+#[cfg_attr(not(feature = "concurrent"), blanket(derive(Ref, Box, Rc)))]
pub trait Parser: LSend {
fn parse(&self, source: &[char]) -> Vec<Token>;
}