web search done right
New spell corrector training. I have just now gotten to the point where we can test this, so it definitely doesn't work yet but must o f the parts should be there.
Mikkel Denkerf30e9131f942
diff --git a/Cargo.lock b/Cargo.lockindex 9646e570..a9572872 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5364,6 +5364,7 @@ checksum = "dcdcd0195a5b871e50926da8e881277f36a4621b3220d85092e7b91cc85f6bd9"dependencies = ["hashbrown 0.12.3","once_cell",+ "serde",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 93615be6..3a9a9bb5 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -106,7 +106,7 @@ urlencoding = "2.1.2"utoipa = {version = "4.0.0", features = ["axum_extras"]}utoipa-swagger-ui = {version = "4.0.0", features = ["axum"]}uuid = "1.1.2"-whatlang = "0.16.0"+whatlang = {version = "0.16.0", features = ["serde"]}[profile.test.package]flate2.opt-level = 3diff --git a/configs/web_spell.toml b/configs/web_spell.tomlnew file mode 100644index 00000000..d92aec62--- /dev/null+++ b/configs/web_spell.toml@@ -0,0 +1,7 @@+languages = ["eng"]+output_path = "data/web_spell"++[warc_source]+folder = "data"+names = ["sample.warc.gz"]+type = "Local"diff --git a/configs/webgraph/profile.toml b/configs/webgraph/profile.tomlindex b56c05d8..880693f4 100644--- a/configs/webgraph/profile.toml+++ b/configs/webgraph/profile.toml@@ -1,7 +1,7 @@-level = "host"graph_base_path = "data/webgraph"+level = "host"[warc_source]+folder = "data"+names = ["sample.warc.gz"]type = "Local"-folder = "data/warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 624ded77..b7aa3b1e 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -138,10 +138,6 @@ name = "abstractive-summarizer"harness = falsename = "build-similarity"-[[bench]]-harness = false-name = "spell-correction"-[[bench]]harness = falsename = "similar-sites"diff --git a/core/benches/spell-correction.rs b/core/benches/spell-correction.rsdeleted file mode 100644index e2b6a65b..00000000--- a/core/benches/spell-correction.rs+++ /dev/null@@ -1,34 +0,0 @@-use criterion::{criterion_group, criterion_main, Criterion};-use stract::{index::Index, searcher::SearchQuery, spell::Spell};--const INDEX_PATH: &str = "data/index";--macro_rules! bench {- ($query:tt, $spell:ident, $c:ident) => {- let mut desc = "correct '".to_string();- desc.push_str($query);- desc.push('\'');- $c.bench_function(desc.as_str(), |b| {- b.iter(|| {- $spell- .correction(&SearchQuery {- query: $query.to_string(),- ..Default::default()- })- .unwrap()- })- });- };-}--pub fn criterion_benchmark(c: &mut Criterion) {- let index = Index::open(INDEX_PATH).unwrap();- let spell = Spell::for_index(&index);-- for _ in 0..100 {- bench!("asdf", spell, c);- }-}--criterion_group!(benches, criterion_benchmark);-criterion_main!(benches);diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 4fe4ad91..5af09039 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -273,7 +273,7 @@ async fn search_metric<B>(if response.status().is_success() {state.counters.search_counter_success.inc();- } else {+ } else if response.status().is_server_error() {state.counters.search_counter_fail.inc();}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 447c9a00..1f23e112 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -448,3 +448,12 @@ pub struct FeedIndexingConfig {pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,}++#[derive(Debug, Deserialize, Clone)]+pub struct WebSpellConfig {+ pub output_path: String,+ pub warc_source: WarcSource,+ pub languages: Vec<whatlang::Lang>,+ pub limit_warc_files: Option<usize>,+ pub skip_warc_files: Option<usize>,+}diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 3db29cf7..8f40c4db 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -29,6 +29,7 @@ pub mod feed_indexer;pub mod indexer;pub mod safety_classifier;pub mod search_server;+pub mod web_spell;mod webgraph;pub mod webgraph_server;diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rsnew file mode 100644index 00000000..30b920fc--- /dev/null+++ b/core/src/entrypoint/web_spell.rs@@ -0,0 +1,162 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::path::Path;++use anyhow::Result;+use fnv::FnvHashMap;+use itertools::Itertools;+use serde::{Deserialize, Serialize};+use tracing::info;+use whatlang::Lang;++use crate::{+ config::{self, WebSpellConfig},+ entrypoint::download_all_warc_files,+ mapreduce::Worker,+ web_spell::{FirstTrainer, FirstTrainerResult, SecondTrainer},+ webpage::Html,+};++pub struct SpellWorker {+ trainer: FnvHashMap<Lang, FirstTrainer>,+}++impl SpellWorker {+ pub fn new<P: AsRef<Path>>(languages: &[Lang], path: P) -> Result<Self> {+ let mut trainer = FnvHashMap::default();++ for lang in languages {+ trainer.insert(+ *lang,+ FirstTrainer::new(path.as_ref().join(lang.to_string()))?,+ );+ }++ Ok(Self { trainer })+ }++ pub fn process(&mut self, job: Job) -> Result<()> {+ let name = job.warc_path.split('/').last().unwrap();++ info!("processing {}", name);++ let source = job.source_config.clone();++ let paths = vec![job.warc_path.clone()];+ let warc_files = download_all_warc_files(&paths, &source);+ tokio::pin!(warc_files);++ for file in warc_files.by_ref() {+ for record in file.records().flatten() {+ let webpage = match Html::parse(&record.response.body, &record.request.url) {+ Ok(webpage) => webpage,+ Err(err) => {+ tracing::error!("error parsing webpage: {}", err);+ continue;+ }+ };++ let lang = match webpage.lang() {+ Some(lang) => *lang,+ None => continue,+ };++ match (webpage.clean_text(), self.trainer.get_mut(&lang)) {+ (Some(text), Some(trainer)) => {+ trainer.add(text);+ }+ _ => continue,+ }+ }+ }++ info!("finished processing {}", name);++ Ok(())+ }++ fn next_training_step(self) -> Result<FnvHashMap<Lang, FirstTrainerResult>> {+ let mut result = FnvHashMap::default();++ for (lang, trainer) in self.trainer {+ result.insert(lang, trainer.next_training_step()?);+ }++ Ok(result)+ }+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct Job {+ pub source_config: config::WarcSource,+ pub warc_path: String,+}++impl Worker for SpellWorker {}++pub fn run(config: WebSpellConfig) -> Result<()> {+ let warc_paths = config.warc_source.paths()?;++ let jobs: Vec<_> = warc_paths+ .into_iter()+ .take(config.limit_warc_files.unwrap_or(usize::MAX))+ .map(|warc_path| Job {+ source_config: config.warc_source.clone(),+ warc_path,+ })+ .collect_vec();++ let num_workers = num_cpus::get();+ let mut handlers = Vec::new();++ for i in 0..num_workers {+ let path = Path::new(&config.output_path).join(format!("{i}"));+ std::fs::create_dir_all(&path)?;+ let mut worker = SpellWorker::new(&config.languages, path)?;++ let jobs = jobs.clone();+ handlers.push(std::thread::spawn(move || {+ for job in jobs.into_iter().skip(i).step_by(num_workers) {+ worker.process(job).unwrap();+ }++ worker.next_training_step().unwrap()+ }));+ }++ let mut combined: FnvHashMap<Lang, Vec<FirstTrainerResult>> = FnvHashMap::default();+ for handler in handlers {+ let trained = handler.join().unwrap();++ for (lang, result) in trained.into_iter() {+ combined.entry(lang).or_default().push(result);+ }+ }++ for (lang, results) in combined {+ let second_trainer = SecondTrainer::new(+ results,+ Path::new(&config.output_path)+ .join("corrector")+ .join(lang.to_string()),+ )?;++ second_trainer.train()?;+ }++ Ok(())+}diff --git a/core/src/lib.rs b/core/src/lib.rsindex e7761ca0..f665d8c0 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -73,13 +73,13 @@ pub mod searcher;mod simhash;pub mod similar_sites;mod snippet;-pub mod spell;#[cfg(feature = "libtorch")]pub mod summarizer;mod tokenizer;#[allow(unused)]mod ttl_cache;pub mod warc;+pub mod web_spell;pub mod webgraph;pub mod webpage;mod widgets;diff --git a/core/src/main.rs b/core/src/main.rsindex 3d69d3ed..74ba767f 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -71,11 +71,15 @@ enum Commands {},/// Deploy the search server.- SearchServer { config_path: String },+ SearchServer {+ config_path: String,+ },/// Deploy the json http api. The api interacts with/// the search servers, webgraph servers etc. to provide the necesarry functionality.- Api { config_path: String },+ Api {+ config_path: String,+ },/// Scrape the Google autosuggest API for search queries.AutosuggestScrape {@@ -107,10 +111,16 @@ enum Commands {alice: bool,},+ // Commands for the live index.LiveIndex {#[clap(subcommand)]options: LiveIndex,},++ // Build spell correction model.+ WebSpell {+ config_path: String,+ },}#[derive(Subcommand)]@@ -385,6 +395,10 @@ fn main() -> Result<()> {.block_on(entrypoint::live_index::serve(config))?}},+ Commands::WebSpell { config_path } => {+ let config: config::WebSpellConfig = load_toml_config(config_path);+ entrypoint::web_spell::run(config)?;+ }}Ok(())diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsindex b2c77b3c..a9cf0a27 100644--- a/core/src/search_prettifier/mod.rs+++ b/core/src/search_prettifier/mod.rs@@ -28,7 +28,7 @@ use crate::{inverted_index::RetrievedWebpage,ranking::{Signal, SignalScore},snippet::TextSnippet,- spell::{self, CorrectionTerm},+ web_spell::{self, CorrectionTerm},webpage::url_ext::UrlExt,};@@ -66,8 +66,8 @@ pub struct HighlightedSpellCorrection {pub highlighted: String,}-impl From<spell::Correction> for HighlightedSpellCorrection {- fn from(correction: spell::Correction) -> Self {+impl From<web_spell::Correction> for HighlightedSpellCorrection {+ fn from(correction: web_spell::Correction) -> Self {let mut highlighted = String::new();let mut raw = String::new();diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex f226cde6..cc449870 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -37,7 +37,6 @@ use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGuse crate::schema::TextField;use crate::search_ctx::Ctx;use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};-use crate::spell::Spell;use crate::webgraph::Node;use crate::webpage::region::Region;use crate::{inverted_index, live_index, Error, Result};@@ -118,7 +117,7 @@ impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {pub struct LocalSearcher<I: SearchableIndex> {index: I,- spell: Option<Spell>,+ // spell: Option<Spell>,entity_index: Option<EntityIndex>,inbound_similarity: Option<InboundSimilarity>,linear_regression: Option<Arc<LinearRegression>>,@@ -151,7 +150,7 @@ whereLocalSearcher {index,- spell: None,+ // spell: None,entity_index: None,inbound_similarity: None,linear_regression: None,@@ -161,7 +160,7 @@ where}pub fn build_spell_dict(&mut self) {- self.spell = Some(Spell::for_index(self.index.guard().search_index()));+ todo!("Spell checker is not implemented yet")}pub fn set_entity_index(&mut self, entity_index: EntityIndex) {@@ -372,11 +371,11 @@ wherelet ctx = guard.inverted_index().local_search_ctx();let inverted_index_result =self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;- let correction = self.spell.as_ref().and_then(|s| s.correction(query));let sidebar = self.entity_sidebar(query);Ok(InitialWebsiteResult {- spell_corrected_query: correction,+ // spell_corrected_query: correction,+ spell_corrected_query: None,websites: inverted_index_result.webpages,num_websites: inverted_index_result.num_hits,has_more: inverted_index_result.has_more,@@ -575,53 +574,4 @@ mod tests {}}}-- #[test]- fn sentence_spell_correction() {- let mut index = Index::temporary().expect("Unable to open index");-- index- .insert(Webpage::new(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever- </body>- </html>- "#- ,- "https://www.example.com",- ).unwrap())- .expect("failed to insert webpage");-- index.commit().unwrap();-- let mut searcher = LocalSearcher::new(index);- searcher.build_spell_dict();-- assert_eq!(- String::from(- searcher- .spell- .as_ref()- .unwrap()- .correction(&SearchQuery {- query: "th best".to_string(),- ..Default::default()- })- .unwrap()- ),- "the best".to_string()- );- assert_eq!(- searcher.spell.as_ref().unwrap().correction(&SearchQuery {- query: "the best".to_string(),- ..Default::default()- }),- None- );- }}diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 2ed4d250..57de3fb2 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -33,7 +33,7 @@ use crate::{search_prettifier::{DisplayedAnswer, DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection,},- spell::Correction,+ web_spell::Correction,webpage::region::Region,widgets::Widget,};diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex 6e80a0a3..263db069 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -18,8 +18,8 @@ use std::ops::Range;use crate::config::SnippetConfig;use crate::query::Query;-use crate::spell::sentence_ranges;use crate::tokenizer::{BigramTokenizer, Normal, Stemmed, Tokenizer, TrigramTokenizer};+use crate::web_spell::sentence_ranges;use crate::webpage::region::Region;use hashbrown::{HashMap, HashSet};use utoipa::ToSchema;diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rsdeleted file mode 100644index 5b913e03..00000000--- a/core/src/spell/dictionary.rs+++ /dev/null@@ -1,442 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::spell::distance::LevenshteinDistance;-use std::cmp::Reverse;-use std::collections::{BTreeMap, BinaryHeap};-use std::hash::{Hash, Hasher};-use std::ops::Deref;-use std::sync::Arc;-use std::{cmp, io};-use thiserror::Error;--pub trait EditStrategy: Send + Sync {- fn distance_for_string(&self, string: &str) -> usize;- fn dist(&self) -> LevenshteinDistance;-}--pub struct LogarithmicEdit {- max_edit_distance: usize,-}--impl LogarithmicEdit {- pub fn new(max_edit_distance: usize) -> Self {- Self { max_edit_distance }- }-}--impl EditStrategy for LogarithmicEdit {- fn distance_for_string(&self, string: &str) -> usize {- let log_value: usize = (string.len() as f32).log2() as usize;- cmp::max(1, cmp::min(log_value, self.max_edit_distance))- }-- fn dist(&self) -> LevenshteinDistance {- LevenshteinDistance::new(self.max_edit_distance)- }-}--#[cfg(test)]-pub struct MaxEdit {- max_edit_distance: usize,-}--#[cfg(test)]-impl MaxEdit {- pub fn new(max_edit_distance: usize) -> Self {- Self { max_edit_distance }- }-}--#[cfg(test)]-impl EditStrategy for MaxEdit {- fn distance_for_string(&self, _: &str) -> usize {- self.max_edit_distance- }-- fn dist(&self) -> LevenshteinDistance {- LevenshteinDistance::new(self.max_edit_distance)- }-}--pub struct DictionaryResult {- pub prob: f64,- pub correction: String,-}--impl Hash for DictionaryResult {- fn hash<H>(&self, state: &mut H)- where- H: Hasher,- {- self.correction.hash(state);- }-}--impl PartialEq for DictionaryResult {- fn eq(&self, other: &Self) -> bool {- self.correction == other.correction- }-}--impl Eq for DictionaryResult {}--#[derive(Error, Debug)]-pub enum DictionaryError {- #[error("Underlying error from FST")]- Fst(#[from] fst::Error),-- #[error("An IO error has occured")]- Io(#[from] io::Error),-}--pub type Result<T> = std::result::Result<T, DictionaryError>;--pub type TermId = u64;--#[derive(PartialEq, Eq, PartialOrd, Ord)]-struct Monogram(TermId);--#[derive(PartialEq, Eq, PartialOrd, Ord)]-struct Bigram(TermId, TermId);--#[derive(PartialEq, Eq, PartialOrd, Ord)]-struct Trigram(TermId, TermId, TermId);--type HeavyMonogram = String;-type HeavyBigram = (String, String);-type HeavyTrigram = (String, String, String);--pub struct DictionaryBuilder {- monograms: BinaryHeap<Reverse<(u64, HeavyMonogram)>>,- bigrams: BinaryHeap<Reverse<(u64, HeavyBigram)>>,- trigrams: BinaryHeap<Reverse<(u64, HeavyTrigram)>>,- top_n: usize,-}--impl DictionaryBuilder {- pub fn new(top_n: usize) -> Self {- Self {- monograms: BinaryHeap::new(),- bigrams: BinaryHeap::new(),- trigrams: BinaryHeap::new(),- top_n,- }- }-- pub fn add_monogram(&mut self, term: HeavyMonogram, freq: u64) {- if self.monograms.len() < self.top_n {- self.monograms.push(Reverse((freq, term)));- } else if self.monograms.peek().unwrap().0 .0 < freq {- let mut worst = self.monograms.peek_mut().unwrap();- *worst = Reverse((freq, term));- }- }-- pub fn add_bigram(&mut self, term: HeavyBigram, freq: u64) {- if self.bigrams.len() < self.top_n {- self.bigrams.push(Reverse((freq, term)));- } else if self.bigrams.peek().unwrap().0 .0 < freq {- let mut worst = self.bigrams.peek_mut().unwrap();- *worst = Reverse((freq, term));- }- }-- pub fn add_trigram(&mut self, term: HeavyTrigram, freq: u64) {- if self.trigrams.len() < self.top_n {- self.trigrams.push(Reverse((freq, term)));- } else if self.trigrams.peek().unwrap().0 .0 < freq {- let mut worst = self.trigrams.peek_mut().unwrap();- *worst = Reverse((freq, term));- }- }-- pub fn build(self) -> Dictionary {- Dictionary::build(- self.monograms- .into_iter()- .map(|Reverse((freq, term))| (term, freq)),- self.bigrams- .into_iter()- .map(|Reverse((freq, term))| (term, freq)),- self.trigrams- .into_iter()- .map(|Reverse((freq, term))| (term, freq)),- )- }-}--#[derive(Clone)]-pub struct Dictionary {- inner: Arc<InnerDictionary>,-}--impl Dictionary {- pub fn build(- monograms: impl Iterator<Item = (String, u64)>,- bigrams: impl Iterator<Item = ((String, String), u64)>,- trigrams: impl Iterator<Item = ((String, String, String), u64)>,- ) -> Self {- let inner = InnerDictionary::build(monograms, bigrams, trigrams);- Self {- inner: Arc::new(inner),- }- }-}--impl Deref for Dictionary {- type Target = InnerDictionary;-- fn deref(&self) -> &Self::Target {- &self.inner- }-}--/// Dictionary that contains term frequency information-pub struct InnerDictionary {- total_freq: u64,- monograms: BTreeMap<Monogram, u64>,- bigrams: BTreeMap<Bigram, u64>,- trigrams: BTreeMap<Trigram, u64>,- terms: BTreeMap<String, TermId>,- rev_terms: BTreeMap<TermId, String>,-}--impl InnerDictionary {- fn build(- monograms: impl Iterator<Item = (String, u64)>,- bigrams: impl Iterator<Item = ((String, String), u64)>,- trigrams: impl Iterator<Item = ((String, String, String), u64)>,- ) -> Self {- let mut dict = InnerDictionary {- total_freq: 0,- monograms: BTreeMap::new(),- bigrams: BTreeMap::new(),- trigrams: BTreeMap::new(),- terms: BTreeMap::new(),- rev_terms: BTreeMap::new(),- };-- dict.load_monograms(monograms);- dict.load_bigrams(bigrams);- dict.load_trigrams(trigrams);-- dict- }-- fn load_monograms(&mut self, monograms: impl Iterator<Item = (String, u64)>) {- for (term, freq) in monograms {- let l = self.terms.len();- let id = *self.terms.entry(term.clone()).or_insert_with(|| l as u64);- self.rev_terms.insert(id, term);- *self.monograms.entry(Monogram(id)).or_insert(0) += freq;- self.total_freq += freq;- }- }-- fn load_bigrams(&mut self, bigrams: impl Iterator<Item = ((String, String), u64)>) {- for ((a, b), freq) in bigrams {- let l = self.terms.len();- let id_a = *self.terms.entry(a.clone()).or_insert_with(|| l as u64);- if self.rev_terms.get(&id_a).is_none() {- self.rev_terms.insert(id_a, a);- }-- let l = self.terms.len();- let id_b = *self.terms.entry(b.clone()).or_insert_with(|| l as u64);- if self.rev_terms.get(&id_b).is_none() {- self.rev_terms.insert(id_b, b);- }-- *self.bigrams.entry(Bigram(id_a, id_b)).or_insert(0) += freq;- }- }-- fn load_trigrams(&mut self, trigrams: impl Iterator<Item = ((String, String, String), u64)>) {- for ((a, b, c), freq) in trigrams {- let l = self.terms.len();- let id_a = *self.terms.entry(a.clone()).or_insert_with(|| l as u64);- if self.rev_terms.get(&id_a).is_none() {- self.rev_terms.insert(id_a, a);- }-- let l = self.terms.len();- let id_b = *self.terms.entry(b.clone()).or_insert_with(|| l as u64);- if self.rev_terms.get(&id_b).is_none() {- self.rev_terms.insert(id_b, b);- }-- let l = self.terms.len();- let id_c = *self.terms.entry(c.clone()).or_insert_with(|| l as u64);- if self.rev_terms.get(&id_c).is_none() {- self.rev_terms.insert(id_c, c);- }-- *self.trigrams.entry(Trigram(id_a, id_b, id_c)).or_insert(0) += freq;- }- }-- #[inline]- pub fn score(&self, before: &[&str], term: &str, after: &[&str]) -> Option<f64> {- self.terms.get(term).map(|id| {- let mut d = 0;- let mut total_score = 0.0;- if let Some(mono) = self.monograms.get(&Monogram(*id)) {- total_score += *mono as f64 / self.total_freq as f64;- d += 1;- }-- if !before.is_empty() {- if let Some(id_before) = self.terms.get(before[before.len() - 1]) {- if let Some(bigram_freq) = self.bigrams.get(&Bigram(*id_before, *id)) {- total_score +=- *bigram_freq as f64 / self.monograms[&Monogram(*id_before)] as f64;- d += 1;- }-- if before.len() > 1 {- if let Some(id_before_before) = self.terms.get(before[before.len() - 2]) {- if let Some(trigram_freq) =- self.trigrams- .get(&Trigram(*id_before_before, *id_before, *id))- {- if let Some(c_freq) =- self.bigrams.get(&Bigram(*id_before_before, *id_before))- {- total_score += *trigram_freq as f64 / *c_freq as f64;- d += 1;- }- }- }- }- }- }-- if !after.is_empty() {- if let Some(id_after) = self.terms.get(after[0]) {- if let Some(bigram_freq) = self.bigrams.get(&Bigram(*id, *id_after)) {- total_score += *bigram_freq as f64 / self.monograms[&Monogram(*id)] as f64;- d += 1;- }-- if after.len() > 1 {- if let Some(id_after_after) = self.terms.get(after[1]) {- if let Some(trigram_freq) =- self.trigrams.get(&Trigram(*id, *id_after, *id_after_after))- {- if let Some(c_freq) = self.bigrams.get(&Bigram(*id, *id_after)) {- total_score += *trigram_freq as f64 / *c_freq as f64;- d += 1;- }- }- }- }- }- }-- if !after.is_empty() && !before.is_empty() {- if let Some(id_after) = self.terms.get(after[after.len() - 1]) {- if let Some(id_before) = self.terms.get(before[before.len() - 1]) {- if let Some(trigram_freq) =- self.trigrams.get(&Trigram(*id_before, *id, *id_after))- {- if let Some(c_freq) = self.bigrams.get(&Bigram(*id_before, *id)) {- total_score += *trigram_freq as f64 / *c_freq as f64;- d += 1;- }- }- }- }- }-- if d == 0 {- 0.0- } else {- total_score / d as f64- }- })- }-- pub fn terms(&self) -> impl Iterator<Item = (&String, &TermId)> {- self.terms.iter()- }-- pub fn term_id(&self, term: &str) -> Option<&TermId> {- self.terms.get(term)- }-- pub fn term(&self, term_id: &u64) -> Option<&str> {- self.rev_terms.get(term_id).map(|s| s.as_str())- }-}--#[cfg(test)]-pub fn build_from_str(text: &str) -> Dictionary {- use itertools::Itertools;- use std::collections::HashMap;- let mut mono_freqs = HashMap::new();- let mut bi_freqs = HashMap::new();- let mut tri_freqs = HashMap::new();-- for word in text.split_ascii_whitespace() {- *mono_freqs.entry(word.to_string()).or_insert(0) += 1;- }-- for (a, b) in text.split_ascii_whitespace().tuple_windows() {- *bi_freqs.entry((a.to_string(), b.to_string())).or_insert(0) += 1;- }-- for (a, b, c) in text.split_ascii_whitespace().tuple_windows() {- *tri_freqs- .entry((a.to_string(), b.to_string(), c.to_string()))- .or_insert(0) += 1;- }-- Dictionary::build(- text.split_ascii_whitespace()- .map(|word| (word.to_string(), mono_freqs[word])),- text.split_ascii_whitespace().tuple_windows().map(|(a, b)| {- (- (a.to_string(), b.to_string()),- bi_freqs[&(a.to_string(), b.to_string())],- )- }),- text.split_ascii_whitespace()- .tuple_windows()- .map(|(a, b, c)| {- (- (a.to_string(), b.to_string(), c.to_string()),- tri_freqs[&(a.to_string(), b.to_string(), c.to_string())],- )- }),- )-}--#[cfg(test)]-mod tests {-- use super::*;-- #[test]- fn test_probability() {- let dict = build_from_str("this is a test test");-- assert_eq!(- dict.score(&[], "this", &[]).unwrap(),- dict.score(&[], "is", &[]).unwrap()- );- assert!(dict.score(&[], "test", &[]).unwrap() > dict.score(&[], "this", &[]).unwrap());- }-}diff --git a/core/src/spell/distance.rs b/core/src/spell/distance.rsdeleted file mode 100644index 615b80b5..00000000--- a/core/src/spell/distance.rs+++ /dev/null@@ -1,112 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};-use std::hash::{Hash, Hasher};--#[derive(Default, Serialize, Deserialize, Debug)]-pub struct LevenshteinDistance {- dist: usize,-}--impl LevenshteinDistance {- pub fn new(distance: usize) -> Self {- LevenshteinDistance { dist: distance }- }-- pub fn distance(&self) -> usize {- self.dist- }-- /// Function taken from https://github.com/febeling/edit-distance/blob/5597816456e7153cf69092f6ab5d0b4edb5e3797/src/lib.rs#L31- ///- /// The [Levenshtein edit distance][wikipedia] between two strings is- /// the number of individual single-character changes (insert, delete,- /// substitute) necessary to change string `a` into `b`.- ///- /// This can be a used to order search results, for fuzzy- /// auto-completion, and to find candidates for spelling correction.- ///- /// This function returns the edit distance between strings `a` and `b`.- ///- /// The runtime complexity is `O(m*n)`, where `m` and `n` are the- /// strings' lengths.- pub fn compare(a: &str, b: &str) -> usize {- let len_a = a.chars().count();- let len_b = b.chars().count();- if len_a < len_b {- return Self::compare(b, a);- }- // handle special case of 0 length- if len_a == 0 {- return len_b;- } else if len_b == 0 {- return len_a;- }-- let len_b = len_b + 1;-- let mut pre;- let mut tmp;- let mut cur = vec![0; len_b];-- // initialize string- for (i, val) in cur.iter_mut().enumerate().take(len_b).skip(1) {- *val = i;- }-- // calculate edit distance- for (i, ca) in a.chars().enumerate() {- // get first column for this row- pre = cur[0];- cur[0] = i + 1;- for (j, cb) in b.chars().enumerate() {- tmp = cur[j + 1];- cur[j + 1] = std::cmp::min(- // deletion- tmp + 1,- std::cmp::min(- // insertion- cur[j] + 1,- // match or substitution- pre + usize::from(ca != cb),- ),- );- pre = tmp;- }- }- cur[len_b - 1]- }-}--#[derive(Debug)]-pub struct EditDistance {- pub dist: usize,- pub edited_string: String,-}--impl PartialEq for EditDistance {- fn eq(&self, other: &Self) -> bool {- self.edited_string == other.edited_string- }-}--impl Eq for EditDistance {}--impl Hash for EditDistance {- fn hash<H: Hasher>(&self, state: &mut H) {- self.edited_string.hash(state);- }-}diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rsdeleted file mode 100644index 6c800843..00000000--- a/core/src/spell/mod.rs+++ /dev/null@@ -1,239 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub mod dictionary;-pub mod distance;-pub mod spell_checker;-pub mod splitter;-pub mod word2vec;--use std::ops::Range;--use itertools::intersperse;-use serde::{Deserialize, Serialize};-use tracing::info;--use crate::index::Index;-use crate::query::parser::Term;-use crate::schema::TextField;-use crate::searcher::SearchQuery;-use crate::{floor_char_boundary, query};--use self::dictionary::DictionaryBuilder;-pub use self::dictionary::{Dictionary, DictionaryResult, EditStrategy, LogarithmicEdit};-pub use self::spell_checker::SpellChecker;-pub use self::splitter::TermSplitter;--#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]-pub struct Correction {- original: String,- pub terms: Vec<CorrectionTerm>,-}--#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]-pub enum CorrectionTerm {- Corrected(String),- NotCorrected(String),-}--impl From<Correction> for String {- fn from(correction: Correction) -> Self {- intersperse(- correction.terms.into_iter().map(|term| match term {- CorrectionTerm::Corrected(correction) => correction,- CorrectionTerm::NotCorrected(orig) => orig,- }),- " ".to_string(),- )- .collect()- }-}--impl Correction {- pub fn empty(original: String) -> Self {- Self {- original,- terms: Vec::new(),- }- }-- pub fn push(&mut self, term: CorrectionTerm) {- self.terms.push(term);- }-- pub fn is_all_orig(&self) -> bool {- self.terms- .iter()- .all(|term| matches!(term, CorrectionTerm::NotCorrected(_)))- }-}--pub fn sentence_ranges(text: &str) -> Vec<Range<usize>> {- let mut res = Vec::new();- let mut last_start = 0;-- // We should really do something more clever than this.- // Tried using `SRX`[https://docs.rs/srx/latest/srx/] but it was a bit too slow.- for (end, _) in text- .char_indices()- .filter(|(_, c)| matches!(c, '.' | '\n' | '?' | '!'))- {- res.push(last_start..end + 1);- last_start = floor_char_boundary(text, end + 2);- }-- res-}--pub struct Spell {- dict: Dictionary,- spell_checker: SpellChecker<LogarithmicEdit>,-}--impl Spell {- pub fn for_index(index: &Index) -> Self {- let dict = Self::build_dict(index);- let spell_checker = SpellChecker::new(dict.clone(), LogarithmicEdit::new(3));-- Self {- dict,- spell_checker,- }- }- fn build_dict(index: &Index) -> Dictionary {- info!("Building spell correction dictionary");- let searcher = index.inverted_index.tv_searcher();- let schema = searcher.schema();- let mut dict = DictionaryBuilder::new(20_000);-- #[allow(unused_assignments, unused_mut)]- let mut limit_terms: Option<usize> = None;- #[cfg(debug_assertions)]- {- limit_terms = Some(100);- }-- for segment in searcher.segment_readers() {- let inv_index = segment- .inverted_index(schema.get_field(TextField::CleanBody.name()).unwrap())- .unwrap();- let term_dict = inv_index.terms();- let mut stream = term_dict.stream().unwrap();- let mut count = 0;- while let Some((term, info)) = stream.next() {- if let Some(limit) = limit_terms {- if count > limit {- break;- }- }-- let term = std::str::from_utf8(term).unwrap();-- if !term.is_empty()- && term- .chars()- .all(|c| c.is_ascii_alphabetic() || c.is_whitespace())- {- dict.add_monogram(term.to_ascii_lowercase(), info.doc_freq as u64);- count += 1;- }- }- }-- dict.build()- }-- fn spell_check(&self, terms: &[String]) -> Option<Correction> {- let mut original = String::new();- let num_terms = terms.len();- let terms = terms.iter().map(|s| s.as_str()).collect::<Vec<_>>();-- for term in &terms {- original.push_str(term);- original.push(' ');- }- original = original.trim_end().to_string();-- let mut possible_correction = Correction::empty(original);-- for i in 0..num_terms {- let before = if i > 0 { &terms[..i] } else { &[] };- let term = &terms[i];- let after = if i < num_terms - 1 {- &terms[i + 1..]- } else {- &[]- };-- match self.spell_checker.correct(before, term, after) {- Some(correction) => possible_correction- .push(CorrectionTerm::Corrected(correction.to_ascii_lowercase())),- None => possible_correction- .push(CorrectionTerm::NotCorrected(term.to_ascii_lowercase())),- }- }-- if possible_correction.is_all_orig() {- None- } else {- Some(possible_correction)- }- }-- fn split_words(&self, terms: &[String]) -> Option<Correction> {- let splitter = TermSplitter::new(&self.dict);-- let mut original = String::new();-- for term in terms {- original.push_str(term);- original.push(' ');- }- original = original.trim_end().to_string();-- let mut possible_correction = Correction::empty(original);-- for term in terms {- let split = splitter.split(term.as_str());- if split.is_empty() {- possible_correction.push(CorrectionTerm::NotCorrected(term.to_string()));- } else {- for s in split {- possible_correction.push(CorrectionTerm::Corrected(s.to_string()))- }- }- }-- if possible_correction.is_all_orig() {- None- } else {- Some(possible_correction)- }- }-- pub fn correction(&self, query: &SearchQuery) -> Option<Correction> {- let terms: Vec<_> = query::parser::parse(&query.query)- .into_iter()- .filter_map(|term| match *term {- Term::Simple(s) => Some(String::from(s)),- _ => None,- })- .map(|s| s.to_ascii_lowercase())- .collect();-- self.spell_check(&terms)- .or_else(|| self.split_words(&terms))- }-}diff --git a/core/src/spell/segment.srx b/core/src/spell/segment.srxdeleted file mode 100644index 39266eab..00000000--- a/core/src/spell/segment.srx+++ /dev/null@@ -1,6813 +0,0 @@-<!---This file is directly copied from-https://github.com/languagetool-org/languagetool/blob/2c91941c436cae9f4c70980b8bb28c9b3817cfbe/languagetool-core/src/main/resources/org/languagetool/resource/segment.srx-and is this thus licensed under GPL v2.1 in accordance with the license of the languagetool repository--->--<?xml version="1.0" encoding="UTF-8"?>-<srx xmlns="http://www.lisa.org/srx20" xmlns:okpsrx="http://okapi.sf.net/srx-extensions" version="2.0">-<header segmentsubflows="yes" cascade="yes">-<formathandle type="start" include="no"></formathandle>-<formathandle type="end" include="yes"></formathandle>-<formathandle type="isolated" include="no"></formathandle>-<okpsrx:options oneSegmentIncludesAll="no" trimLeadingWhitespaces="no" trimTrailingWhitespaces="no" useJavaRegex="yes" useIcu4JBreakRules="no" treatIsolatedCodesAsWhitespace="no"></okpsrx:options>-<okpsrx:sample language="nl" useMappedRules="yes">De organisatie Doe! is een rare.</okpsrx:sample>-<okpsrx:rangeRule></okpsrx:rangeRule>-</header>-<body>-<languagerules>-<languagerule languagerulename="Greek">-<!--κ.λπ. - και λοιπά-->-<rule break="no">-<beforebreak>\bκ\.λπ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--π.χ. - παραδείγματος χάριν-->-<rule break="no">-<beforebreak>\bπ\.χ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--months-->-<rule break="no">-<beforebreak>\b(Ιαν|Φεβ|Μα[ϊρ]|Απρ|Ιου[νλ]|Αυγ|Σεπ|Οκτ|Νοε|Δεκ)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!;…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!;…]['"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!;…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Polish">-<!--includes v"ad." misused as abbreviation-->-<rule break="no">-<beforebreak>\b[Aa]dw?\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bafr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bakad\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Aa]l\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bam\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bamer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\barch\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Aa]rt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bartyst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bastr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\baustr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbałt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbdb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbł\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbr\.\p{Pe}?\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbry[gt]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcentr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bces\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bchem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bchiń\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bchir\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bc\.k\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bc\.o\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcyg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcyw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcyt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bczes\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bczw?\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Cc]d\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bczyt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bćw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bćwicz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdaw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdcn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdekl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdemokr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdet\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdiec\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdł\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--5 dol., dop. red., dot.-->-<rule break="no">-<beforebreak>\bdo[tlp]\.\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdost\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdosł\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bh\.c\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bds\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bduszp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdypl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\begz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bekol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bekon\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\belektr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bew\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfab\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfarm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfot\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgastr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgeogr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgeol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgimn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgłęb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgodz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgórn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgosp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgram\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhist\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhiszp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Hh]r\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhot\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bid\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bin\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bim\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\biron\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkard\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkatol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bk\.k\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--kol., kl.-->-<rule break="no">-<beforebreak>\bko?l\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bk\.p\.a\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkpc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bk\.p\.c\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkpt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bk\.r\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkrak\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bk\.r\.o\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkryt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkult\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blaic\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\błac\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bniem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bwoj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Nn][bp]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpo[lw]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bm\.in\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--pt., ps.-->-<rule break="no">-<beforebreak>\b[Pp][ts]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcdn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Rr]y[cs]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Tt]zw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btzn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Zz]ob\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsłow\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[^s]\.pl\.\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}+</afterbreak>-</rule>-<!--pl., pw., pn.-->-<rule break="no">-<beforebreak>\bp[wnl]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bang\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--ub. rok i ul. Dobra-->-<rule break="no">-<beforebreak>\bu[lb]\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bal\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--ok. 30 proc. lub ok. trzydzieści procent-->-<rule break="no">-<beforebreak>\b[Oo]k\.\s</beforebreak>-<afterbreak>[\p{Ll}\d]</afterbreak>-</rule>-<!--Include the incorrect form "tyś." (5 tyś. zł), 4,5tys.-->-<rule break="no">-<beforebreak>\b[\d,\.]*ty[sś]\.\p{Pe}?\s</beforebreak>-<afterbreak>[\p{Ll}\d]+</afterbreak>-</rule>-<!--nr. 1 (z błędem)-->-<rule break="no">-<beforebreak>\b[Nn]r\.\s</beforebreak>-<afterbreak>\d</afterbreak>-</rule>-<!--wł., ww.-->-<rule break="no">-<beforebreak>\bw[wł]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bur\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bżyd\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bżarg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bżyw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bwy[łdm]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[bu]p\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bwyst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\b(?iu)mazeł\stow\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Tt]ow\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bo\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([sS]p|st|[Ss]półdz|społ|spółgł|[Ss]to[łw])\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzew\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzewn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzdr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzazw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzast\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzaw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzał\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzal\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzam\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzak\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzakł\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzagr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzach\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Aa]dw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ll]ek\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmed\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Mm]ec\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Dd]oc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--dyw., dyr.-->-<rule break="no">-<beforebreak>\b[Dd]y[wr]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ii]nż\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(?iu)mgr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--dh (druh), dr-->-<rule break="no">-<beforebreak>\b[dD][hr]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--p. Nowak-->-<rule break="no">-<beforebreak>\b[pP]\.\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Rr]ed\.\)?\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(?iu)pro[fk]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhab\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Pp]łk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Nn]a|[Pp]o)dkom\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[kK]s\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(?iu)gen\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(?iu)por\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Rr]eż\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Pp]rzyp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bp\.n\.e\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdyr\.\smuz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--msza św. lub św. Paweł; śp. Jadwiga Jagiełłowa-->-<rule break="no">-<beforebreak>\b[śŚ][pwW]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bw\.\s</beforebreak>-<afterbreak>((?:X{0,2})(?:V?I{0,3}|I[VX]))(?<=[XVI]+)</afterbreak>-</rule>-<!--I poł. XIX w.-->-<rule break="no">-<beforebreak>\bII?\społ\.\s</beforebreak>-<afterbreak>[IVX]+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\betc\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--These can appear at the end of sentence, so watch out for the next letter: it shouldn't be uppercase.-->-<rule break="no">-<beforebreak>\bit[dp]\.\s</beforebreak>-<afterbreak>[„”"]?\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bproc\.\)?\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}\p{Lu}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[rwn]\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bit[dp]\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcdn\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ss]zer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjw\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bn\.e\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bw\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--nt., nn.-->-<rule break="no">-<beforebreak>\bn[tn]\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[dm]l\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdag\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[cd]?m\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Zz][Łł]\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Gg][rR]\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[d]?kg\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bml[nd]\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnpl\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpkt\.\s</beforebreak>-<afterbreak>[\p{Ll}\d]</afterbreak>-</rule>-<!--str. 12-->-<rule break="no">-<beforebreak>\bstr\.\s</beforebreak>-<afterbreak>[\p{Ll}\d]</afterbreak>-</rule>-<!--tab. 12-->-<rule break="no">-<beforebreak>\b[Tt]ab\.\s</beforebreak>-<afterbreak>[\p{Ll}\d]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\btel\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[ptw]g\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcos\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcosec\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsec\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsin\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\brkm\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bust\.\s</beforebreak>-<afterbreak>\d</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpar\.\s</beforebreak>-<afterbreak>\d</afterbreak>-</rule>-<!--lit. a, lit. a)-->-<rule break="no">-<beforebreak>\blit\.\s</beforebreak>-<afterbreak>\p{Ll}\p{Pe}?\p{P}?\s</afterbreak>-</rule>-<!--od pon. do-->-<rule break="no">-<beforebreak>\b[Pp]on\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--od sob. do pt.-->-<rule break="no">-<beforebreak>\b[Ss]ob\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--ba! po prostu głupi!-->-<rule break="no">-<beforebreak>\bba!\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--pok. 131, poz. 124-->-<rule break="no">-<beforebreak>\bpo[zk]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bop\.\s</beforebreak>-<afterbreak>cit\.</afterbreak>-</rule>-<!--oo. Dominikanie-->-<rule break="no">-<beforebreak>\b[Oo][Oo]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Ludwik Cz. został aresztowany-->-<rule break="no">-<beforebreak>\b([CDSR]z|Ch)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--Na s. 112 jest opis orgii, która odbyła się w r. 2005 wg Janusza S. (l. 45).-->-<rule break="no">-<beforebreak>\b[rls]\.\s</beforebreak>-<afterbreak>[1-9]+</afterbreak>-</rule>-<!--1800-700 l. p.n.e.-->-<rule break="no">-<beforebreak>[\d-–]+\sl\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--(od 1945 r.-->-<rule break="no">-<beforebreak>\((?:od|do|w)\s[1-9]\d*\sr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\d+\sr\.\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bn\.e\.\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bt\.\s</beforebreak>-<afterbreak>\d+</afterbreak>-</rule>-<!--i s. Alicji, o s. Rafaeli-->-<rule break="no">-<beforebreak>\b[,uoi]\ss\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Najśw. Sakrament-->-<rule break="no">-<beforebreak>\b[Nn]ajśw\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--na os. Koziniec-->-<rule break="no">-<beforebreak>\b[Nn]a\sos\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\barab\.\s</beforebreak>-<afterbreak>[^\x00-\x80]+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bT\.</beforebreak>-<afterbreak>Love\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpl\.</beforebreak>-<afterbreak>Open[oO]ffice</afterbreak>-</rule>-<!--20 ha. ziemi na Mazurach-->-<rule break="no">-<beforebreak>\bha\.\s</beforebreak>-<afterbreak>[\p{Ll}]</afterbreak>-</rule>-<!--min. 30 zł lub cena min. od 30 zł-->-<rule break="no">-<beforebreak>\bmin\.\s</beforebreak>-<afterbreak>[\p{Ll}\d]</afterbreak>-</rule>-<!--W 1991 r. Józef Stalin nie żył.-->-<rule break="no">-<beforebreak>\bW\s\d{4}\sr\.\s</beforebreak>-<afterbreak>[\p{Lu}-–—„"]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>\.\p{L}+\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--single lowercase chars at the end of sentence except for r, w, n-->-<rule break="no">-<beforebreak>[\s\(\[][\p{L}&&[^rwn]]\.\s+</beforebreak>-<afterbreak>\p{Ll}\p{Ll}|\p{Lu}[\p{Punct}\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>["”']\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[!?]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\p{Ps}][!?]+[\p{Pe}] </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\s\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\p{Ll}\.\s?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\.\p{Lu}\p{Ll}\.\s?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\s(?<!\.)[„"\p{Ps}]?[A-Z]\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}\p{Ll}\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>^[„"]?[A-ZŚĆŻŹ]\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}\p{Ll}\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\(\p{Lu}?\p{Ll}+\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Józef S. (45 l.)-->-<rule break="no">-<beforebreak>\s[A-Z]\.\s</beforebreak>-<afterbreak>\(\d</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bDz\.\s?U\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--heuristic: no vowels = abbreviation-->-<rule break="no">-<beforebreak>[\p{Ll}&&[^aeouiyęąó]][\p{Ll}&&[^aeouiyęąó]]+\.\s</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<!--Nie jesteś Rosjaninem? - spytał przedstawiciel okręgu.-->-<rule break="no">-<beforebreak>[\.!?…]+['"\p{Pe}\u00BB\u2019\u201D\u203A\u0002]*\s</beforebreak>-<afterbreak>[\p{Ps}-–—]\s?\p{Ll}</afterbreak>-</rule>-<!--This is a text ("with a small remark!") that continues.-->-<rule break="no">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\u0002]*\p{Pe}\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--p. n.e. (błędny podział wiersza)-->-<rule break="no">-<beforebreak>p\.\s</beforebreak>-<afterbreak>n\.\s?e\.</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\p{Pe}\u00BB\u2019\u201D\u203A\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['»"”\p{Pe}]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}\.]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>: </beforebreak>-<afterbreak>[—\-–] \p{Lu}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="English">-<rule break="no">-<beforebreak>[\u00A0\s]</beforebreak>-<afterbreak>\n</afterbreak>-</rule>-<rule break="no">-<beforebreak>[a-zA-Z][!\?][\s\u00A0]</beforebreak>-<afterbreak>\)[\s\u00A0][a-zA-Z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>Yahoo![\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[A-Z]\.[A-Z]\.</beforebreak>-<afterbreak>[A-Z]\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bA\.</beforebreak>-<afterbreak>I\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bS\.</beforebreak>-<afterbreak>I\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bL\.</beforebreak>-<afterbreak>A\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bU\.</beforebreak>-<afterbreak>[SK]\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bI\.</beforebreak>-<afterbreak>S\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bM\.</beforebreak>-<afterbreak>Z\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(https?|ftp|file|chrome|chromium|android|(chrome|moz)\-extension):///?[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+\.(com|net|org|info|de|es|edu|co|eu|nl|io|cn|uk|gov|biz|ca|tk|ru|br|jp|pl|be|dev|co|fr|dk|se)(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[nN]o\.[\s\u00A0]</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bP[Hh]\.[\s\u00A0]?</beforebreak>-<afterbreak>D\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Aa]vg|[Ee]d|pp|[Vv]iz|i\.?[\s\u00A0]*e|[Vvol]|[Rr]col|maj|Lt|[Ff]ig|[Ff]igs|[Vv]iz|[Vv]ols|[Aa]pprox|[Ii]ncl?|[Aa]cc|Pres|[Cc]orp|[Ee]x|[Cc]onn|[Dd]ept|[Ll]tda|[Mm]in|[Mm]ax|[Gg]ovt|[Rr]etd|Ing|lb|lbf|ft|c\.?[\s\u00A0]*f|vs|dia|lbs|\d+-(:?oz|kc|in|h[rp]|ml)|M?sec)\.[\s\u00A0]</beforebreak>-<afterbreak>[^\p{Lu}]|I</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(hr)\.[\s\u00A0]</beforebreak>-<afterbreak>[^\p{Lu}]|I</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Vv]ol|[Ff]ig|[Dd]ef|[Ee]q|[Ll]em|[Pp]rop|[Tt]hm)s?\.[\s\u00A0]</beforebreak>-<afterbreak>\p{N}|[IXV]+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Ff]ig|[Dd]ef|[Ee]q|[Ll]em|[Pp]rop|[Tt]hm)s?\.[\s\u00A0]</beforebreak>-<afterbreak>\(\p{N}\)</afterbreak>-</rule>-<rule break="no">-<beforebreak>(…|\.\.\.)[\s\u00A0]?\)[\s\u00A0]</beforebreak>-<afterbreak>[^\p{P}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[?!.…]["”]\)[\s\u00A0]</beforebreak>-<afterbreak>[a-z].*</afterbreak>-</rule>-<rule break="no">-<beforebreak>(…|\.\.\.)[\s\u00A0]?\?\)[\s\u00A0]</beforebreak>-<afterbreak>[^\p{P}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\be\.g\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Vv]s\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(pp|PP)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\be[sx]p\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--"Etc." can end the sentence, so we check for the uppercase letter after it.-->-<rule break="no">-<beforebreak>\b[Ee]tc\.[\s\u00A0]</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Bb]tw|BTW)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bJan\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bFeb\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bMar\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bApr\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bJu[nl]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bAug\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bSept?\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bOct\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bNov\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bDec\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>(?i)FRITZ!</beforebreak>-<afterbreak>(?i)Box</afterbreak>-</rule>-<rule break="no">-<beforebreak>ID.</beforebreak>-<afterbreak>3|4|Buzz|Crozz</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bP[Hh]\.?[\s\u00A0]?[Dd]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(P[hH][dD]|BSc|BEng|BComp|BArch|MSc|MEng|MComp)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bLL\.[\s\u00A0]?[BM]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Eng\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bLL\.[\s\u00A0]?</beforebreak>-<afterbreak>[BM]\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Sc\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Comp?\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Arch\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.?[\s\u00A0]?(Sc|Eng|Comp|Arch)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bet\b[\s\u00A0]\bal\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(a(?:bbrev|uth|bl|bsol|bstr|cc|ccus|dv|dvb|dvs|gst|lt|phet|pp|ppos|cc|dj|djs|rch|rt|ttrib)|A(?:bbrev|uth|bd|berd|berdeensh|bol|borig|bp|br|bridg|bridgem|bsol|bst|bstr|cad|cc|ccept|ccomm|ccompl|ccs|cct|ccts|chievem|dd|ddit|ddr|dm|dmin|dmir|dmon|dmonit|dv|dvancem|dvert|dvoc|dvt|dvts|erodynam|eronaut|ff|ffect|fr|gric|lch|lg|lleg|llit|lm|lph|mer|nal|nalyt|nat|nc|necd|ng|ngl|nim|nn|nniv|nnot|nsw|nt|nthrop|nthropol|ntiq|poc|pol|pp|ppl|pplic|rch|rchaeol|rchipel|rchit|rgt|rith|rithm|rrangem|rtic|rtific|rtill|ssemb|ssoc|ssyriol|str|strol|stron|stronaut|tt|ttrib|ustral|uth|utobiog|utobiogr|yrsh|rab)|B(?:acteriol|edford|edfordsh|elg|erks|erksh|erw|erwicksh|ibliogr|iochem|iog|iogr|iol|ks|ord|ot|raz|rit|ucks|uild|ull|ur)|b(?:ef|etw)|c(?:ent|ollect|olloq|ompar|ompl|onc|oncr|omp|onj|ons|onst|ontempt|orresp|pd|ontr)|C(?:ontradict|ontrib|ontrov|onv|onvent|onversat|onvoc|ornw|oron|orr|orresp|ounc|ourtsh|raniol|raniom|rim|rit|rt|rts|ryptogr|rystallogr|umb|umberld|umbld|ycl|ytol|ollect|onn|al|alc|alend|alif|alligr|amb|ambr|ampanol|anad|anterb|artogr|atal|atech|ath|ent|eram|ert|ertif|hamb|har|harac|has|hem|hesh|hr|hron|hronol|hrons|inematogr|irc|lass|lassif|limatol|lin|oll|olloq|om|omb|ombs|omm|ommandm|ommend|ommerc|ommiss|ommonw|ommunic|omp|ompan|ompar|ompend|ompl|ompos|onc|onch|oncl|onf|onfid|onfl|onfut|ongr|ongreg|ongress|onsc|onsecr|onsid|onsol|onstit|onstr|ontemp|ontempl|ontend|ontent|ontin)|d(?:at|em|ial|im|yslog|ef|eriv|erog)|D(?:au|eb|eclar|ed|ef|eliv|emonstr|ep|epred|epredat|erbysh|escr|evel|evonsh|ial|ict|iffic|irect|is|isc|iscipl|iscov|iscrim|iscuss|iss|istemp|istill|istrib|iv|ivers|oc|octr|omest|urh)|e(?:tym|tymol|uphem|xc|ast|llipt|mph|rron)|E(?:val|vang|ven|vid|vol|xalt|xam|xch|xec|xerc|xhib|xped|xper|xplan|xplic|xplor|xpos|tymol|ccl|ccles|col|con|din|dinb|duc|dw|gypt|gyptol|lectr|lectro-magn|lectro-physiol|lem|liz|lizab|mb|mbryol|ncycl|ng|ngin|nglishw|nq|nt|nthus|ntom|ntomol|nzymol|pil|pisc|pist|pit|quip|ss|ssent|stabl|thnol)|f(?:em|req|ut|am|amil)|F(?:ifesh|ootpr|orfarsh|ortif|ortn|ound|ragm|ratern|riendsh|und|urnit|ab|am|arew)|G(?:ard|astron|az|eo|eog|eogr|eol|eom|eomorphol|er|lac|lasg|los|loss|louc|loucestersh|osp|ram|ynaecol)|g(?:erund|en)|H(?:aematol|ampsh|andbk|ants|eb|en|er|erb|eref|ereford|erefordsh|ertfordsh|ierogl|ist|istol|om|orol|ort|osp|ouseh|ousek|usb|ydraul|ydrol)|hist|I(?:nd|ndustr|nfl|nnoc|norg|nq|nst|ntell|ntellect|nterc|nterl|nternat|nterpr|chth|cthyol|deol|dol|llustr|mag|mpr|naug|nclos|nd|nstr|tal|ntro|ntrod|nv|nvent|nvertebr|nvestig|nvestm|nvoc|rel|mmunol)|i(?:nt|nterj|nterrog|ntr|ntrans|mp|mperf|mpers|mpf|mprop|nstr|nd|ndef|ndic|ndir|nfin|nfl|ron|rreg|mit)|J(?:ahrb|ap|as|rnl|rnls|urisd|urisdict|urispr|ustif|ustific)|joc|K(?:ent|ingd|nowl|pr)|L(?:ab|anc|ancash|ancs|ang|angs|at|d|ds|ect|eechd|eg|eicest|eicester|eicestersh|eics|et|ett|ex|ibr|imnol|incolnsh|incs|ing|inn|it|ithogr|ithol|iturg|ond)|m(?:asc|ed|etaphor|idl|ispr|od)|M(?:ach|ag|agn|an|anagem|anch|anip|anuf|ath|eas|easurem|ech|ed|edit|em|erc|erch|etall|etallif|etallogr|etamorph|etaph|eteorol|eth|etrop|ex|ich|icrobiol|icrosc|il|ilit|in|ineral|isc|iscell|od|onum|orphol|SS|tg|unic|unif|unim|us|yst|yth|ythol)|n(?:once-wd|orth|om)|N(?:arr|arrat|at|aut|av|avig|eighb|erv|eurol|eurosurg|ewc|ewspr|onconf|orf|orthamptonsh|orthants|orthumb|orthumbld|orthumbr|orw|orweg|otts|ucl|umism|on-conf)|o(?:ccas|pp|rig|bj|bl|bs)|O(?:bs|bserv|bstet|bstetr|ccas|ccup|ccurr|ceanogr|ff|ffic|kla|nt|phthalm|phthalmol|ppress|pt|rac|rd|rg|rig|rkn|rnith|rnithol|rthogr|utl|xf|xfordsh|xon|bed|bj)|p(?:ass|erf|ers|ersonif|honet|hr|op|lur|oet|ref|rep|riv|rob|oss|pl|ple|ples|rec|red|redic|ron|ronunc|rop|rov|ropr|seudo-arch|seudo-dial|seudo-Sc|erh|res)|P(?:eriodontol|redict|rerog|sych|sychoanal|sychoanalyt|sychol|sychopathol|ubl|urg|erf|alaeobot|alaeogr|alaeont|alaeontol|araphr|arasitol|arl|arnass|ath|athol|eculat|enins|ers|ersec|erthsh|etrogr|etrol|harm|harmaceut|harmacol|hil|hilad|hilol|hilos|hoen|honol|hotog|hotogr|hrenol|hys|hysiogr|hysiol|ict|oet|ol|olit|olytechn|op|orc|ort|osth|ostm|ott|ract|ref|reh|rehist|resb|reserv|rim|rinc|rint|robab|robl|roc|rod|rol|rov|rovid|rovinc|rovis|ronunc|rop|ros)|Qld|q(?:uot|uots)|r(?:edupl|eg|epr|het|efash|efl|el)|R(?:adiol|eas|eb|ebell|ec|eclam|ecoll|edempt|ef|efl|efus|efut|eg|egic|egist|egr|el|elig|eminisc|emonstr|enfrewsh|eprod|ept|epub|es|esid|et|etrosp|evol|het|ich|om|oxb|oy|udim|uss)|s(?:ing|outh|pec|tr|ubj|ubjunct|ubord|ubseq|ubst|uff|uperl|yll)|S(?:ubj|uff|ubscr|ubscript|uppl|upplic|uppress|urg|urv|ymmetr|ymp|yst|pan|ask|at|ax|cand|ch|ci|cot|cotl|cript|culpt|eismol|el|elect|er|erm|ess|ettlem|ev|hakes|haks|heph|hetl|hropsh|oc|ociol|om|onn|pec|pecif|pecim|pectrosc|taff|tafford|taffordsh|taffs|tand|tat|tatist|tratigr|truct|tud)|t(?:echn|rans|ransf|ransl)|T(?:ransl|ransubstant|rav|reas|reat|reatm|rib|rig|rigonom|rop|roub|roubl|ypog|ypogr|axon|rans|echn|echnol|el|elecomm|elegr|eleph|eratol|erminol|errestr|est|extbk|heat|heatr|heol|heoret|hermonucl|hes|opogr|rag)|U(?:niv|rin)|u(?:nkn|nstr|lt|su)|U(?:nnat|noffic|tilit)|V(?:ac|aledict)|v(?:ar|arr|ars|bl|bs|ulg)|V(?:eg|enet|ertebr|et|ic|ict|ind|indic|irg|irol|oc|ocab|ol|oy|ulg)|W(?:estm|estmld|estmorld|estmrld|ill|ilts|iltsh|is|isd|kly|ks|onderf|orc|orcestersh|orcs|rit|arwicksh)|west|Y(?:earbk|ng|orks|orksh|rs)|Z(?:eitschr|oogeogr|ool))\.\b</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Atty|Sg?t|[SG]en|Ft|Gov|Hon|Prof|Mr?s|Mt|[DMJS]r|Col|Maj|L(ieu)?t|Brig|Capt|Cmdr|Cmnd|Revd?|Rep)\.[[\s\u00A0]\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Atty|Sg?t|[SG]en|Gov|Hon|Prof|Mr?s|[DMJS]r|Col|Maj|L(ieu)?t|Brig|Capt|Cmdr|Cmnd|Revd?|Rep)\.[\s\u00A0][A-Z]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Drs|Messrs|Mmes)\.[\s\u00A0]</beforebreak>-<afterbreak>(and[\s\u00A0])|\p{Lu}\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcf\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bI(nc|NC)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bCorp\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bBros\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bDist\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bCo\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bo'clock[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfo'c'sle[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bLtd\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Ps}[!?]+\p{Pe} </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.!?…]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>["”'’][\s\u00A0]*</beforebreak>-<afterbreak>[\s\u00A0]*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”][\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.[\s\u00A0]</beforebreak>-<afterbreak>\p{L}\.[\s\u00A0]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Lu}\p{L}+[\s\u00A0]v\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Lu}\p{L}+</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[^,][\s\u00A0]\p{L}{2}\.[\s\u00A0]</beforebreak>-<afterbreak>\p{N}+\)[\s\u00A0]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\bOK\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s\u00A0](?!(on|it|of|to|be|by|at|he|we|so|do|if|up|my|me|us|go|am))\p{L}{1,2}\.[\s\u00A0]</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.[\s\u00A0]\p{Lu}\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\p{Lu}\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[^\.][\s\u00A0][A-Z]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(:?Blvd|Ave|Mts?)\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(?:Kan|Ill|M[ai]ss)\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\(\p{Ll}+\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>i\.e\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\s\u00A0]\p{L}[\.!?…][\s\u00A0]</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Romanian">-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<rule break="no">-<beforebreak>["”']\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s]\p{L}{1,2}\.\s</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[!?]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\p{Ps}][!?]+[\p{Pe}] </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\b(etc|șamd)\.\s</beforebreak>-<afterbreak>[A-Z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(pag|leg|art)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ian|febr?|mart?|apr|iu[nl]|aug|sept?|oct|nov|dec)\.\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdpdv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(etc|șamd)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(M)\.</beforebreak>-<afterbreak>Ap\.N\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(M)\.Ap\.</beforebreak>-<afterbreak>N\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Dd]l|[Dd]-na|[Dd]vs|[Pp]t)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Dd]l|[Dd]-na|[Dd]vs|[Pp]t)\.\s[A-Z]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['»"”\p{Pe}]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Dutch">-<rule break="no">-<beforebreak>\b(sp|SP)</beforebreak>-<afterbreak>\.[aA]\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\s[.]</beforebreak>-<afterbreak>[Nn][Ee][Tt](\b|-)</afterbreak>-</rule>-<rule break="no">-<beforebreak>[.?!][’'"]</beforebreak>-<afterbreak> [a-z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(https?|ftp|file|chrome|chromium|android|(chrome|moz)\-extension):///?[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+\.(com|net|org|info|de|es|edu|co|eu|nl|io|cn|uk|gov|biz|ca|tk|ru|br|jp|pl)(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(blz|pag|fig)\.\s</beforebreak>-<afterbreak>[0-9]</afterbreak>-</rule>-<!--Abbrevs that can happen in sentence and at end-->-<rule break="no">-<beforebreak>\b(enz|etc|zat|ambt|al|ver|art|wed|lab|bv|Bros)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s(la|do|del)\sMar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Ge?n|Ex|Le?v|Nu?m|D(eu)?t|Jo?z|Ri|R[ei]cht|Sa?m|Ko?n|Kr[on]{0,2}|Neh?|Est?|Jb|Ps|Spr?|Pr[ed]{0,2}|H(oog)?l|Je?s|Je?r|Kl(aagl)?|Ez(ech)?|Da?n|Ho?s|Jl|Am|Ob|Mc|Mi[ch]{0,2}|Nah?|Hk|Hab|Zf|[SZ]ef|Ha?g|Zc|Zach|Ma?l|Ma?t|Mk|Mar|Lk|Jh|H(an)?d|Ro?m|Kor|Ga?l|Ef|Fp|Fil|Ko|[CK]ol|Th|Th?e[s]{1,2}|Tm|Ti?t|Fm|Fil(em)?|Hb|Hebr?|Jk|Ja[ck]|Pe?tr?|Joh|Jud|Op(enb)?|Wijsh|Tob|Sir|Bar|Makk)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Drs|Art|Afr|Am|Ar|Br|Cie|Comp|Dhr|(Prof\.)?[Dd]r|Em|Fa|Kon|Stb)\.\s</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Stb)\.\s</beforebreak>-<afterbreak>[0-9]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Mm]ej|[Mm]evr|[Mm]rs|[Mm]s|[Mm]gr|[Mm]w|Ndl|Ned|Nl|No|Prof|[Ss]ecr|Chr|Jac|[Ww]ed|Zr)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Sr|St|Ued|Vz|aanh|aanw|aardew|aardr)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(abs|abstr|adj|adm|[Aa]fb|[Aa]fd|afk|afl|milj|zgn|plv|bvb|afm|evt|exp|vs)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ald|alg|amb|anat|antrop|apoth)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\seen\sprof\.\s</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(alc|bro|opm|acc)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(arch|archeolbc|bep|betr|bez|bibl|bijl|[Bb]ijv)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(bijz|bw|ca|cat|centr|cf|cfr|cmpl)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(conf|ct|dal|derg|dhr|dir|div|dra|drs|ds)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Ee]d|em|ev|[Ee]xcl|[Ff]a|[Ff]am|[fF]ig|fin|fl|fr)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(geb|[Gg]em|get|gld|id|[Ii]ncl|ind|inf|ing|intern|inz|ir|jhr|jkvr)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(jl|jr|kr|kt|lic|ll|lt|lw|max|[Mm]evr|mi|[Mm]in|mld)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(mln|[Mm]r|[Mm]w|nl|no|nr|nrs|ob|obl|ong|onov)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(opm|org|ov|[Pp]ag|par|penn|([1-3][\.e]?)[\s]?pers|plm|plv)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(prov|pseud|psych|qty|red|ref|resp|soc|st|tab|tel|temp|prof|tk)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([A-Z]|Adr|Chr|Fr|Fred|IJ|Jac|Joh|Ph|St|Th|Tj|v|v\.(\s)?d)\.(\s)?</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[vn]\.\s</beforebreak>-<afterbreak>Chr</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(uitsl|vgl|vnl|vnw|voorz|ww|zat|[Zz]elfst|zgn?)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(mm|cm|km|ml|mg|kg|h|kW|kg|mW)\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\b(mm|cm|km|ml|mg|kg|h|kW|kg|mW)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Ps}[!?]+\p{Pe} </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.!?…]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>["”']\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s]\p{L}{1,2}\.\s</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\s\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[^\.]\s[A-Z]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\.\p{Lu}\p{Ll}\.\s?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<!--a number with a dot before a lowercase char-->-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[.!?…][’'"]\s</beforebreak>-<afterbreak>[a-z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[.][.]\s</beforebreak>-<afterbreak>[a-z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>SP[.]</beforebreak>-<afterbreak>A</afterbreak>-</rule>-<rule break="no">-<beforebreak>Warner Bros\.</beforebreak>-<afterbreak>[a-z]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s'[2-9][.]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\s[A-Z].+!\s</beforebreak>-<afterbreak>[a-z]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[.!?…][’'"\u00BB\u2019\u201D\u203A\u00AB\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[.!?…][’'"\u00BB\u2019\u201D\u203A\u00AB\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-<!-- names with exclamation mark in them -->-<rule break="no">-<beforebreak>\p{L}!</beforebreak>-<afterbreak>[^ ]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\)</beforebreak>-<afterbreak>\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>\.</beforebreak>-<afterbreak>\)</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\)\.</beforebreak>-<afterbreak>[A-Z]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\.\)</beforebreak>-<afterbreak>[A-Z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmax\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[?!.]['"\u00BB\u2019\u201D\u203A\u00AB\p{Pe}\u0002]\s</beforebreak>-<afterbreak>[A-Z][a-z]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[?!.]\s</beforebreak>-<afterbreak>['"\u00BB\u2019\u201D\u203A\u00AB\p{Pe}\u0002][A-Z][a-z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>"[A-Z][.]\s</beforebreak>-<afterbreak>[a-z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[A-Z][a-z].*sz[.]\s</beforebreak>-<afterbreak>[a-z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>De n[.]\s</beforebreak>-<afterbreak>[a-z]|[XIV]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[A-Z]{2,5}[.]</beforebreak>-<afterbreak>[A-Z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\.\.</beforebreak>-<afterbreak>" [a-z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\sBTW\.</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Slovak">-<rule break="no">-<beforebreak>\b(Bc|Mgr|RNDr|PharmDr|PhDr|JUDr|PaedDr|ThDr|Ing|MUDr|MDDr|MVDr|Dr|ThLic|PhD|ArtD|ThDr|Dr|DrSc|CSs|prof)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Oo]br|[Čč])\.\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\babl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\babsol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\badj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\badmin\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Aa]dr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\badv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\badvok\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bafr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bak\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bakad\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bakc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bakuz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bet\b\s\bal\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bal\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\balch\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bamer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\banat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Aa]ngl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\banglosas\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\banorg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bap\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bapod\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\barch\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\barcheol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\barchit\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\barg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bart\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bastr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bastrol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bastron\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\batp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\batď\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Aa]ustr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\baut\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Bb]elg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Bb]ibl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbiol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbot\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbud\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbás\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbýv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcest\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bchem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcirk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bcsl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Čč]s\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdep\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdet\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdial\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdiaľ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdipl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdistrib\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdokl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdosl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdopr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdram\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bduš\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdvojčl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdór\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bekol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bekon\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bel\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\belektr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\belektrotech\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\benerget\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bepic\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\best\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\betc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\betonym\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\beufem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ee]uróp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bev\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bevid\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bexpr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfa\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfam\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfarm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfeud\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfil\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfilat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfiloz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfi\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfon\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bform\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfot\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ff]r\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ff]ranc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfraz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfut\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfyz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfyziol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgarb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgen\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgenet\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgenpor\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgeod\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgeogr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgeol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgeom\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgerm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Gg]r\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Gg]réc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgréckokat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhebr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bherald\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhist\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\bhl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhlav\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhosp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhromad\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhud\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhypok\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bident\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bi\.?e\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bident\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bimp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bimpf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bindoeur\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\binf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\binform\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\binstr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bint\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\binterj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\binšt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\binštr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\biron\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Jj]ap\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjaz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjedn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjuhoamer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjuhových\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjuhozáp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bjuž\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Kk]anad\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkanc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkapit\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkpt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkart\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkatastr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bknih\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkniž\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkomp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkonj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkonkr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkozmet\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkrajč\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkresť\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkuch\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blatinskoamer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blek\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blex\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blingv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blit\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blitur\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blog\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blok\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Mm]ax\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Mm]aď\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmedzinár\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmest\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmetr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Mm]il\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Mm]in\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bminer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bml\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmld\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmod\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmytol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnapr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Nn]ar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnasl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnedok\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bneg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnegat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bneklas\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Nn]em\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bneodb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bneos\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bneskl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnesklon\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnespis\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnespráv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bneved\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnež\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bniekt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bniž\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnom\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnáb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnákl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnámor\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnár\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bobch\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bobj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bobv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bobyč\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bobč\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bobčian\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bodb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bodd\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bods\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bojed\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Oo]kr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bopt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bopyt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\borg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bos\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bosob\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bot\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bovoc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpart\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpejor\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpers\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(pf|Pf|P\.f|p\.f)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bPlk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpod\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpodst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpokl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpolit\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpolitol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpolygr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpomn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpopl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpor\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bporad\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bporov\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bposch\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpotrav\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpouž\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpoz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpozit\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpoľ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpoľno\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpoľnohosp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpoľov\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpošt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpož\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprac\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpredl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpren\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprep\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpreuk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Pp]riezv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprivl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprof\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpráv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpríd\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpríj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprík\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpríp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprír\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprísl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpríslov\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpríč\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpsych\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpubl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpís\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpísm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpôv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\brefl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\breg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\brep\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bresp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\brozk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\brozlič\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\brozpráv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Rr]oč\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bryb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\brádiotech\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\brím\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsamohl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsemest\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsev\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bseveroamer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bseverových\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bseverozáp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bskr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bskup\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bSloven\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsoc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsoch\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsociol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ss]pol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bspoloč\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bspoluhl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bspráv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bspôs\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstarogréc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstarorím\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bs\.r\.o\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstor\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstredoamer\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstredoškol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsubj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsubst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsuperl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsúkr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsúp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsúvzť\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Tt]al\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btech\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Tt]el\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btelef\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bteles\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btelev\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bteol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btrans\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bturist\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btuzem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btypogr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btzn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btzv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bukaz\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Uu]l\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bumel\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\buniv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bust\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bved\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvedľ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bverb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bveter\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvin\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bviď\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvod\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvodohosp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bp?nl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvulg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvyj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvys\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvysokoškol\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvzťaž\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvôb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvých\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvýd\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvýrob\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvýsk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvýsl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvýtv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvýtvar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvýzn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvčel\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvš\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvšeob\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzahr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzariad\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzast\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzastar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzastaráv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzdravot\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzdruž\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzjemn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzlat\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Zz]n\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzool\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzried\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzáhr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzák\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzákl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzám\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzáp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzápadoeur\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bzázn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\bázij\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\búzem\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\búčt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bčast\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Čč]es\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bčl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bčísl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bživ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfak\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ss]lov\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bKr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bp\.n\.l\.\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Jj]r\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(P\.\s?S|p\.\s?s|P\.\s?s)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Ps}[!?]+\p{Pe} </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.!?…]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>["”']\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s]\p{L}{1,2}\.\s</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\s\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[^\.]\s[A-Z]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\p{Ll}\.\s?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s]\p{L}{1,2}\.\s</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Icelandic">-<!-- Numbers -->-<rule break="no">-<beforebreak>\b[nN]o\.\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[nN][rR]\.\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<!-- Date/time stuff -->-<rule break="no">-<beforebreak>\b\p{N}+\.\s</beforebreak>-<afterbreak>\b(janúar|febrúar|mars|apríl|maí|júní|júlí|ágúst|september|október|nóvember|desember)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{N}+\.\s</beforebreak>-<afterbreak>\b(jan|feb|mar|apr|maí|jún|júl|ágú|sep|sept|okt|nóv|des)</afterbreak>-</rule>-<!-- (final) single char abbreviations x. xxx. xx.x. but not 'íáóæ' -->-<rule break="no">-<beforebreak>\b[a-z[^íáóæ]]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!-- Misc A -->-<rule break="no">-<beforebreak>\bab\.fn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\ba\.fn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bafs\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bal\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\balm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\balg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bandh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bath\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\baths\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\batr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bao\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bau\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\baukaf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc Á -->-<rule break="no">-<beforebreak>\báfn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\báhrl\.s\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\báhrs\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bákv\.gr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bákv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc B -->-<rule break="no">-<beforebreak>\bbh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bbls\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc D -->-<rule break="no">-<beforebreak>\bdr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc E -->-<rule break="no">-<beforebreak>\be\.Kr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bet\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bef\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\befn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bennfr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\beink\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bend\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\be\.st\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\berl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc F -->-<rule break="no">-<beforebreak>\bfél\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfskj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bf\.hl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfísl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-ZÁ-Þ][a-zá-þ]+fj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfo\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bforl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfrb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfrl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfrh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfrt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfsl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfsh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfs\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfsk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bf\.Kr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bft\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfyrrn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bfyrrv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc G -->-<rule break="no">-<beforebreak>\bgerm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bgr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc H -->-<rule break="no">-<beforebreak>\bhdl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhdr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhlsk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhljsk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhljv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhljóðv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-ZÁ-Þ][a-zá-þ]+hr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhvk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bholl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bHos\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhöf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bhrl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc Í -->-<rule break="no">-<beforebreak>\bísl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc K -->-<rule break="no">-<beforebreak>\bkaf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkap\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bKhöfn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bklst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkgúrsk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bkvk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc L -->-<rule break="no">-<beforebreak>\bleturbr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blh\.nt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blh\.þt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\blo\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bltr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc M -->-<rule break="no">-<beforebreak>\bmlja\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmljó\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmillj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmms\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bm\.fl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmiðm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmgr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bmín\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc N -->-<rule break="no">-<beforebreak>\bnf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnhm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnmgr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bno\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnúv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bnt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc O -->-<rule break="no">-<beforebreak>\bo\.áfr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bo\.m\.fl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bohf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bo\.fl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bo\.s\.frv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc Ó -->-<rule break="no">-<beforebreak>\bófn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bób\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bóákv\.gr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bóákv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc P -->-<rule break="no">-<beforebreak>\bpfn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bPR\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bpr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc R -->-<rule break="no">-<beforebreak>\bRitstj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bRvík\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bRvk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc S -->-<rule break="no">-<beforebreak>\bsamb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsamhlj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsamn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsamn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsbr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsek\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsérn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsfn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsfn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bs\.hl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bskv\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bso\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bss\.us\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bs\.st\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsamþ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsbr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bshlj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsign\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bskál\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bst\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bst\.s\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bstk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bsþ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc T -->-<rule break="no">-<beforebreak>\bteg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btbl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btfn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btvíhlj\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btvt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\btill\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bto\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc U -->-<rule break="no">-<beforebreak>\bumr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\buh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bus\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\buppl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc Ú -->-<rule break="no">-<beforebreak>\bútg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc V -->-<rule break="no">-<beforebreak>\bvb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bVf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvkf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bVl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvlf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvmf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b8vo\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvsk\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvth\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Misc Þ -->-<rule break="no">-<beforebreak>\bþt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþjs\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþgf\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþlt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþolm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþm\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþml\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bþýð\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Yes breaks -->-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Russian">-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<!-- capital char abbreviations А. Б. В. -->-<rule break="no">-<beforebreak>\b[А-ЯЁ]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Z]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- N(ame).Family -->-<rule break="no">-<beforebreak>\b[А-ЯЁ]\.</beforebreak>-<afterbreak>[А-ЯЁ][а-яё]+</afterbreak>-</rule>-<!-- N(ame).S(urname).Family -->-<rule break="no">-<beforebreak>\b[А-ЯЁ]\.[А-ЯЁ]\.</beforebreak>-<afterbreak>[А-ЯЁ][а-яё]+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!-- date/time -->-<rule break="no">-<beforebreak>\b[0-9]+(гг|г)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[XVILMC]+(в|вв)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-9]+(\.|:)[0-9][0-9]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-9]+(\.|:)[0-9][0-9](\.|:)[0-9][0-9]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Measures -->-<rule break="no">-<beforebreak>\b[0-9]+(м|мм|см|дм|л|км|га|кг|т|г|мг)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-9]+(руб|Руб|тыс|Тыс|трлн|млн|млрд)\.\s</beforebreak>-<afterbreak>\b[0-9]+</afterbreak>-</rule>-<!-- other abbreviations -->-<rule break="no">-<beforebreak>\b(бульв|г|д|доп|др|е|зам|Зам|и|им|инд|исп|Исп)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(англ|в|вв|га|гг|гл|гос|грн|дм|долл|е|ед)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(к|кап|кав|кв|кл|кол|комн|куб|л|лиц|лл|м|макс)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(кг|км|коп|л|лл|м|мг|мин|мл|млн|Млн|млрд|Млрд|мм)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(н|наб|нач|неуд|нем|ном|о|обл|обр|общ|ок|ост|отл|п|пер|Пер|перераб|пл|пос|пр|пром|просп|Просп|проф|Проф)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(р|ред|Рис|рус|с|сб|св|См|см|сов|соч|соц|спец|ср|ст|стр|т|тел|Тел|тех|тов|тт|туп)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(руб|Руб|тыс|Тыс|трлн)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(уд|ул|уч|физ|х|хор|э|Эл|эл)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ч|чел|шт|экз)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„“][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\u00AB][\.!?…][\u00BB]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>["”'\u00BB]\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<!-- break -->-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\u005D\u005D\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['»"”\p{Pe}]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Default">-<rule break="yes">-<beforebreak>\p{L}[\.!?…]\u00A0+\s+</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\p{L}[\.!?…]\s+\u00A0+</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\u2029</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--for smooth interoperability with Anaphraseus translation plugin for OpenOffice.org-->-<rule break="yes">-<beforebreak></beforebreak>-<afterbreak><0\}</afterbreak>-</rule>-<!--Anaphraseus segment start marker-->-<rule break="yes">-<beforebreak>\{0></beforebreak>-<afterbreak></afterbreak>-</rule>-<!--support srt files-->-<rule break="yes">-<beforebreak>\d{2}:\d{2}:\d{2},\d{3}\r?\n</beforebreak>-<afterbreak></afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="ByLineBreak">-<rule break="yes">-<beforebreak>\r?\n</beforebreak>-<afterbreak></afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="ByTwoLineBreaks">-<rule break="yes">-<beforebreak>\r?\n\s*\r?\n[\t]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[.!?]\u00A0\r?\n</beforebreak>-<afterbreak></afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Slovenian">-<rule break="no">-<beforebreak>\b[dD]r\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bitd\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bitn\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[šŠ]t\.\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bd\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[jJ]an\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[fF]eb\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[mM]ar\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[aA]pr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[jJ]u[ln]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[aA]vg\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[sS]ept?\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[oO]kt\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[nN]ov\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[dD]ec\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[tT]j\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[nN]pr\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[sS]l\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[oO]p\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[gG]l\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[oO]z\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bprev\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bdipl\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bing\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[pP]rim\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[cC]f\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-9]+(\.|:)[0-9][0-9](\.|:)[0-9][0-9]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-3][0-9]+(\.|:)[0-9][0-9](\.|:)[0-9][0-9]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-9]+\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[XVILMC]+\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[gG]l\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Catalan">-<rule break="no">-<beforebreak>Yahoo![\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\w['’][nNtT]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\.\[\d+\][\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- initials: A. C. Jones. Problem: [...] d'Alfons I. Ell era [...] -->-<rule break="no">-<beforebreak>\b[A-ZÀÉÈÍÓÒÚ]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Abbreviations that cannot finish sentences-->-<rule break="no">-<beforebreak>\b(dc|(?iu)(n|Mr|C|Dr|Dra|Dra\. Ma|Sta\. Ma|E|Emm|Emma|Excm|Excma|Hble|I|Il·lm|Il·lma|Il·ltre|Im|Ima|Mgfc|Mgfca|Mn|R|Rev|Sr|Sra|Sres|Srs|St|Sta|a|abr|abs|acad|add|adj|adm|admdor|admdora|admtiu|admtiva|adv|ag|agl|agr|agron|agròn|aj|ajud|al|alim|amb|ampl|ant|ap|apmt|apnt|apr|aprox|apt|arm|arq|arqueol|arquit|assign|assoc|atm|aut|aux|av|b|batx|bda|bibl|bl|bnc|butll|bxs|c|calef|cartogr|cat|catedr|catol|cf|cia|cin|cint|circul|cit|climat|col|col·l|compt|cons|constr|cont|contr|conv|corp|corr|cpl|cpt|cró|ct|cte|ctra|cts|d|dept|derog|des|desp|dg|dip|disp|distr|div|dj|dl|doc|drec|ds|dt|dta|dte|dupl|dv|e|econ|ed|ef|entl|esc|esp|espf|esq|ex|exc|exp|exped|ext|f|fac|fca|febr|fig|figs|fra|gen|gov|gral|i|imp|impr|impt|inc|insp|inst|int|inv|j|jul|jur|jurispr|leg|llic|loc|ltda|làm|merc|mil·l|màx|mín|neg|nov|nre|núm|o|oct|op|p|pàg|pàgs|paq|par|pda|pg|pl|pobl|pol|ppda|ppt|pral|prev|prof|progr|prov|pta|ptes|ptge|pvt|pàg|quadr|quint|r|rbla|ref|reg|rev|secr|serv|sgt|sotsp|subsp|supl|supt|t|tel|telegr|tit|trad|trans|transcr|transf|trav|tripl|trv|tt|tèc|univ|urb|v|var|veg|venc|vid|vig|vocab|vs|x|àt|íd))\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Abbreviations that can finish sentences -->-<rule break="no">-<beforebreak>\b(s|ca)\.[\s\u00A0]</beforebreak>-<afterbreak>[XIV]+\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(min|m|ca)\.[\s\u00A0]</beforebreak>-<afterbreak>[0-9]+\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Cc]ap|[Aa]rts?|pp|[Vv]ol)\.[\s\u00A0]</beforebreak>-<afterbreak>[XIV\d]+\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Ee]ds?|[Cc]oords?|\d+(r|n|t|è|é|a|rs|ns|es)|seg|masc|fem|sing|pl|adj|adv|g|kg|m|km|cm|ha|u|h|hrs|s|ss|alt|cant|cast|cert|com|dir|gr|nom|parc|pres|set|Sr|Jr|Admón|Adm|Inc|Co|Hnos|Vda|[VU]d[s]?)\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>[\-¡¿«»"'\u2018\u201C\p{Ps}\u2012\u2013\u2014\u2015\u2053]*\p{Ll}</afterbreak>-</rule>-<!-- Any word in acronyms like U.S.A.F or F. B. I. or C. or c.s.p. or p. e. -->-<rule break="no">-<beforebreak>\b(\p{L}\.)+[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Any word in acronyms like EE.UU. or BB. DD. -->-<rule break="no">-<beforebreak>\b([\p{Lu}]{2}\.)+[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}{2}\.[\s\u00A0]?</beforebreak>-<afterbreak>\p{Lu}{2}</afterbreak>-</rule>-<rule break="no">-<beforebreak>EE\.[\s\u00A0]?UU\.[\s\u00A0]?</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- max min etc -->-<rule break="no">-<beforebreak>\b([Ee]tc|m[aáà]x|m[ií]n|aprox|\d+o)\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Composed abbrev. -->-<rule break="no">-<beforebreak>\bet al\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Units -->-<rule break="no">-<beforebreak>\b([Pp]ta[s]?|K[gm][s]|[mc]?[gmls]|[Hh](rs)?)\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Ellipsis: ... lowercase -->-<rule break="no">-<beforebreak>[^\s\u00A0](\Q...\E|…)[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- (enum...) -->-<rule break="no">-<beforebreak>\b(\Q...\E|…)[\p{Pe}»"’”][\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- pero ¡ah! no estaba-<rule break="no">-<beforebreak>\b¡\p{L}+![\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>- -->-<rule break="yes">-<beforebreak>[\.…][\u00BB\u2019\u201D\u203A"'\u0002]*[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\b[\p{L}'’·\-]+[\p{Pf}\p{Pe}\u00BB\u2019\u201D\u203A"'\u0002]*[\.:!?…]+[\s\u00A0]</beforebreak>-<afterbreak>[¡¿«»"'\u2018\u201C"\p{Ps}]*\p{Lu}\p{L}*</afterbreak>-</rule>-<!-- paragraphs with opening "»" in dialogs-->-<rule break="yes">-<beforebreak>[\.:!?…»]+[\s\u00A0]</beforebreak>-<afterbreak>»[^\u00A0\s\.:!?…]</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Spanish">-<rule break="no">-<beforebreak>¿[^?]+:[\s\u00A0]</beforebreak>-<afterbreak>.</afterbreak>-</rule>-<rule break="no">-<beforebreak>Yahoo![\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>40dB.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\.\[\d+\][\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- initials: A. C. Jones. Problem: [...] de Alfons I. Él era [...] -->-<rule break="no">-<beforebreak>\b[A-ZÀÉÈÍÓÒÚ]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Ellipsis: ... lowercase -->-<rule break="no">-<beforebreak>[^\s\u00A0](\Q...\E|…)[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- (enum...) -->-<rule break="no">-<beforebreak>\b(\Q...\E|…)[\p{Pe}»"’”][\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Abbreviations that can finish sentences -->-<rule break="no">-<beforebreak>\b(s|ca)\.[\s\u00A0]</beforebreak>-<afterbreak>[XIV]+\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(min|m|ca)\.[\s\u00A0]</beforebreak>-<afterbreak>[0-9]+\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Cc]ap|[Aa]rts?|pp|[Vv]ol|p|[Pp][aá]gs?|ps)\.[\s\u00A0]</beforebreak>-<afterbreak>[XIV\d]+\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(\d+(r|er|n|ero|era|mo|ma|vo|va|no|na|to|ta|do|da|h|hr|gr|grs|o|a)s?|g|kg|m|km|cm|ha|u|h|hrs|H|HR|HRS|s|ss|alt|cant|cast|cert|com|dir|gr|nom|parc|pres|set|Sr|Jr|Admón|Adm|Inc|Co|Hnos|Vda|[VU]d[s]?)\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>[\-¡¿«»"'\u2018\u201C\p{Ps}\u2012\u2013\u2014\u2015\u2053]*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(https?|ftp|file|chrome|chromium|android|(chrome|moz)\-extension):///?[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+\.(com|net|org|info|de|es|edu|co|eu|nl|io|cn|uk|gov|biz|ca|tk|ru|br|jp|pl)(\.|\b)</afterbreak>-</rule>-<!-- Abbreviations that cannot finish sentences-->-<rule break="no">-<beforebreak>\b((?iu)(en|febr|mzo|abr|my|jun|jul|ag|agt|set|sept|setbre|oct|nov|novbre|dic|dicbre))\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(dc|(?iu)(n|[Aa]yto|Mr|C|Dr|Dra|E|Emm|Emma|Excm|Excma|Hble|I|Il·lm|Il·lma|Il·ltre|Im|Ima|Mgfc|Mgfca|Mn|R|Rev|Sr|Sra|Sres|Sras|Srs|St|Sta|a|abr|abs|acad|add|adj|adm|admdor|admdora|admtiu|admtiva|adv|ag|agl|agr|agron|agròn|aj|ajud|al|alim|amb|ampl|ant|ap|apmt|apnt|apr|aprox|apt|arm|arq|arqueol|arquit|assign|assoc|atm|aut|aux|av|b|batx|bda|bibl|bl|bnc|butll|bxs|c|calef|cartogr|cat|catedr|catol|cf|cia|cin|cint|circul|cit|climat|col|col·l|compt|cons|constr|cont|contr|conv|corp|corr|cpl|cpt|cró|ct|cte|ctra|cts|d|dept|derog|des|desp|dg|dip|disp|distr|div|dj|dl|doc|drec|ds|dt|dta|dte|dupl|dv|e|econ|ed|ef|entl|esc|esp|espf|esq|ex|exc|exp|exped|ext|f|fac|fca|febr|fig|figs|fra|gen|gov|gral|i|imp|impr|impt|inc|insp|inst|int|inv|j|jul|jur|jurispr|leg|llic|loc|ltda|làm|merc|mil·l|màx|mín|neg|nov|nre|núm|o|oct|op|p|pàg|pàgs|paq|par|pda|pg|pl|pobl|pol|ppda|ppt|pral|prev|prof|progr|prov|pta|ptes|ptge|pvt|pàg|quadr|quint|r|rbla|ref|reg|rev|secr|serv|sgt|sotsp|subsp|supl|supt|t|tel|telegr|tit|trad|trans|transcr|transf|trav|tripl|trv|tt|tèc|univ|urb|v|var|veg|venc|vid|vig|vocab|vs|x|àt|íd))\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Aa]vda|[Pp][ol]|Pl?za|[Aa]dm|[Dd]pto|Sr|Mr|Srta|ej)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Dña|Dr[a]?|Sra|Sto|S(ri)?ta|Ldo|Ing|Prof|Excmo|Ilmo|Mgfco|admdor|admdora)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Aa]rt|[Cc]ód|[Ss]ecc|[Tt]ít)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Ee]d(it)?|[Nn]o|n|[Nn]úm|[Pp]ág|p|c|\d+er)|[V\.]gr\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Abbreviations that can finish sentences -->-<rule break="no">-<beforebreak>\b([Ee]ds?|[Cc]oords?|grs?|Sr|Jr|Admón|Inc|Co|Hnos|Vda|[VUuv]d[s]?)\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>[\-¡¿«»"'\u2018\u201C\p{Ps}\u2012\u2013\u2014\u2015\u2053]*\p{Ll}</afterbreak>-</rule>-<!-- Any word in acronyms like U.S.A.F or F. B. I. or C. or c.s.p. or p. e. -->-<rule break="no">-<beforebreak>\b(\p{L}\.)+[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Any word in acronyms like EE.UU. or BB. DD. -->-<rule break="no">-<beforebreak>\b([\p{Lu}]{2}\.)+[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}{2}\.[\s\u00A0]?</beforebreak>-<afterbreak>\p{Lu}{2}</afterbreak>-</rule>-<rule break="no">-<beforebreak>EE\.[\s\u00A0]?UU\.[\s\u00A0]?</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- max min etc -->-<rule break="no">-<beforebreak>\b([Ee]tc|m[aá]x|m[ií]n|aprox|\d+o)\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Composed abbrev. -->-<rule break="no">-<beforebreak>\bet al\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Units -->-<rule break="no">-<beforebreak>\b([Pp]ta[s]?|K[gm][s]|[mc]?[gmls]|[Hh](rs)?)\.[\p{Pe}\p{Pf}\p{Pd}"']*[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.…][\u00BB\u2019\u201D\u203A"'\u0002]*[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\b[\p{L}'’·\-]+[\p{Pf}\p{Pe}\u00BB\u2019\u201D\u203A"'\u0002]*[\.:!?…]+[\s\u00A0]</beforebreak>-<afterbreak>[¡¿«»"'\u2018\u201C"\p{Ps}]*\p{Lu}\p{L}*</afterbreak>-</rule>-<!-- paragraphs with opening "»" in dialogs-->-<rule break="yes">-<beforebreak>[\.:!?…»]+[\s\u00A0]</beforebreak>-<afterbreak>»[^\u00A0\s\.:!?…]</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="German">-<rule break="no">-<beforebreak>\b(https?|ftp|file|chrome|chromium|android|(chrome|moz)\-extension):///?[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+\.(com|net|org|info|de|es|edu|co|eu|nl|io|cn|uk|gov|biz|ca|tk|ru|br|jp|pl)(\.|\b)</afterbreak>-</rule>-<!--support simple lists in markdown style-->-<rule break="yes">-<beforebreak>\r?\n[\u00A0\s]*[-*]+[\u00A0\s]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Split at e.g. "1a. Und ..." -->-<rule break="yes">-<beforebreak>\d+[a-z]\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<!-- Don't split at e.g. "d. h." -->-<rule break="no">-<beforebreak>[^-\p{L}'’/°]\p{L}[\.!?…]['|"|“|«|\)|\]|\}]?[\u00A0\s]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>([Dd](as|er|ie|iese[rsmn]?|en|em)|[kmsd]?ein(e[rsnm]?)?|am|fürs|ins|zum|im|am|zur) \d+\.[\u00A0\s]+</beforebreak>-<afterbreak>[A-ZÄÖÜ].*</afterbreak>-</rule>-<rule break="no">-<beforebreak>Ust.</beforebreak>-<afterbreak>Id</afterbreak>-</rule>-<rule break="no">-<beforebreak>Prof.</beforebreak>-<afterbreak>Dr</afterbreak>-</rule>-<rule break="no">-<beforebreak>Dr.</beforebreak>-<afterbreak>iur|med|oec|phil|rer|theol</afterbreak>-</rule>-<rule break="no">-<beforebreak>(?i)FRITZ!</beforebreak>-<afterbreak>(?i)Box</afterbreak>-</rule>-<rule break="no">-<beforebreak>ID.</beforebreak>-<afterbreak>3|4|Buzz|Crozz</afterbreak>-</rule>-<rule break="no">-<beforebreak>[1-3]\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>Liga|Bundesliga|(Fußball|Handball|Basketball)(-B|b)undesliga</afterbreak>-</rule>-<rule break="no">-<beforebreak>\d+\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>Klässler[sn]?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bP[Hh]\.</beforebreak>-<afterbreak>D\.</afterbreak>-</rule>-<!-- Don't split at e.g. "U.S.A." -->-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't split after a white-space followed by a single letter followed- by a dot followed by another whitespace. e.g. " p. " -->-<rule break="no">-<beforebreak>[\u00A0\s]\p{L}\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!-- Don't split at "bla bla... yada yada" -->-<rule break="no">-<beforebreak>[\[\(]?\.\.\.[\]\)]?[\u00A0\s]{1,2}</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Don't split [.?!] when they're quoted -->-<rule break="no">-<beforebreak>['"„][\.!?…]['"“«»][\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't break after quote unless there's a capital letter- e.g.: "That's right!" he said. -->-<rule break="no">-<beforebreak>["'“«»][\u00A0\s]{1,2}</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- e.g. "Das ist . so." - assume one sentence. -->-<rule break="no">-<beforebreak>[\u00A0\s]([\.!?]{1,3}|…)['|"|“|«|\)|\]|\}]?[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Numbers, dates e.g. "3.10. datiert" -->-<rule break="no">-<beforebreak>\b\d+\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<!-- z.B. "Das hier ist ein(!) Satz." -->-<rule break="no">-<beforebreak>[\(\[][!?]{1,3}[\]\)][\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- z.B. "Das hier ist (genau!) ein Satz." -->-<rule break="no">-<beforebreak>[!?]{1,3}[\)\]][\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- z.B. "bla (...) blubb" -> kein Satzende -->-<rule break="no">-<beforebreak>[\(\)\[\]][\u00A0\s]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- don't split at cases like "Friedrich II. wird auch..." -->-<rule break="no">-<beforebreak>[\u00A0\s ][IVX]+\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>[^\p{Lu}]+</afterbreak>-</rule>-<!-- don't split at cases like "im 13. oder 14. Jahrhundert" -->-<rule break="no">-<beforebreak>\d+\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>(und|oder|bis)[\u00A0\s]</afterbreak>-</rule>-<!-- einige deutsche Monate, vor denen eine Zahl erscheinen kann,- ohne dass eine Satzgrenze erkannt wird- (z.B. "am 13. Dezember" -> keine Satzgrenze) -->-<rule break="no">-<beforebreak>\d+\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>Januar|Jänner|Februar|März|Merz|April|Mai|Ju[ln]i|August|September|Oktober|November|Dezember</afterbreak>-</rule>-<rule break="no">-<beforebreak>\d+\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>J[aä]n|Febr?|Mär|Apr|Mai|Ju[nl]|Aug|Sept?|Okt|Nov|Dez</afterbreak>-</rule>-<rule break="no">-<beforebreak>(Jan|Jän|Febr?|Mär|Apr|Mai|Ju[nl]|Aug|Sept?|Okt|Nov|Dez)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>\d\d(\d\d)?</afterbreak>-</rule>-<!-- ähnliche Fälle außerhalb der Monatsnamen -->-<rule break="no">-<beforebreak>\d+\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>Amtsperiode|Breitengrads?|Breitengrades|Jubiläum|Jhd?|Jhdts?|Konferenz|(Jahres|Partei)(-K|k)onferenz|Längengrade?s?|Tags?|Tages|(Jahres|Spiel|Partei|Geburts)tag|(Jahres|Spiel|Partei|Geburts)tages|(Jahres|Spiel|Partei|Geburts)tags|Jahrhunderts?|Jahrtausend|Platz|Platzes|Lebensjahrs?|Lebensjahres|Lochs?|Loches|Grads|Grades|Obergeschoss|Stock(werk)?s?|Etage|Klasse|Runde|Bezirk|Etappe|Staffel|Sinfonie</afterbreak>-</rule>-<!-- English abbreviations - but these work globally for all languages -->-<rule break="no">-<beforebreak>\b(Mrs?|No|pp|St|no|Sr|Jr|[Ss]ek|Bros|etc|[Bb]tw|vs|esp|[Ff]ig|Jan|Feb|Mar|Apr|Ju[nl]|Aug|Sept?|O[ck]t|Nov|Dec|PhD|BSc|BEng|BComp|BArch|al|cf|Inc|Ms|MEng|MSc|MComp|Gen|Sen|Prof|Corp|Co|co|Ltd|Buchst)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Latin abbreviations - but these work globally for all languages -->-<rule break="no">-<beforebreak>\b(spp?)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- German abbreviations -->-<rule break="no">-<beforebreak>\b(betr|Geb|Stk|ggü|Mag|mtl|[Pp]arl|versch|[Ss]tellv|d|Übers|usw|[Bb]zw|Ab[hkst]|[Aa]bzü?gl|[Ll]tda|[Ee]inschl|[Vv]mtl|Ev|bezgl|Abzw|[Vv]sl|ahd|Akk|aktual|[Öö]ffentl|prof|allg|alltagsspr|altdt|alttest|amerikan|Anh|Ank|Anm|Art|[Aa]utom|Auftragsnr|Az|Bat|bayr|Bde?|bearb|Bed|Bem|bes|bez|Bez|Bhf|bspw|btto|bw|Dtl|Dez)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(cts?|[Cc]a|chem|chin|Chr|cresc|[Dd]at|desgl|ders|dgl|Dipl|Dir?|Doz?|durchg|durchges|Dr|dt|ebd|Ed|[Ee]igt?l|akt|[Ee]ngl|Erg|al|et[cw]|Etw|ev|[Ee]vtl?|[Ee]xkl|Expl|Exz)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bDipl\.-[A-Z][a-z]{2,4}\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\u00A0\s]Sc\.[\u00A0\s]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ff|Fa|fachspr|fam|fem|Fem|Fr|franz|[Ff]rz?|[Aa]ltfranz|frdl|Frl|Fut|Gd|gebr?|Gebr|geh|geleg|gen|Gen|germ|gesch|ges|get|ggf|Ggf|Ggs|ggT|Gr|[Gg]rds|griech)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(hebr|hg|hl|Hrsg|Hg|hist|hochd|hochspr|Hptst|Hr|hrsg|Allg|IdNr|ill|[Ii]nkl|[Ii]ncl|[Ee]hem|Ind|Inf|Ing|ital|Tr|jap|Jb|Jg|Jhd?|Jhdts?|jmd[mns]?|jur|Kap|kart|kath|kfm|kaufm|Kfm|kgl|Kl|Konj|königl|Krs?|Kto)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([A-ZÖÄÜ][a-zöäüß]+nr|tel|[Gg]em|Pat|prov|Betr|lat|lfd|Lit|lt|Lz|Mask|mask|max|Mrd|mdal|me[dt]|phil|mhd|Mio?|mio|mind?|Mo|mod|nachm|nördlBr|neutr|Nhd|Nom|Nrn?|Num|Obj|od|dgl|offz)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Part|Per[fs]|Pfd|Pl(ur)?|pl|Plusq|Pos|pp|Prä[ps]|Prät|Pro[vf]|rd|reg|resp|Rhld|rit|Sa|südl|Br|se[ln]|Sept|Sing|sign|So|sog|Sp|Std?|stacc|Str|stud|Subst|sva|svw|sZ)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>([A-ZÖÄÜ][a-zöäüß]+str)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\d+\.\d+\.[\u00A0\s]</beforebreak>-<afterbreak>[\-–][\u00A0\s]\d+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Tel|teilw|Temp|trans|Tsd|übertr|übl|ff|überarb|ugs|univ|unveränd|urspr|USt|UST|USt\-IdNr|[Aa][bn]schl|sw|kl|[Gg]r|vgl|vll|Vll|vlt|Vlt|vllt|Vllt|Vgl|Vol|vollst|vorm|Vp|Vs|vs|wesentl|voraussichtl|[Rr]echts?staatl|[Ss]taatl|wg|Whg|Hd|Ztr|zus|Zus|zzt?|zzgl|zB|zb|Zz|Zt|zw|Min|Bzgl|bzgl|bezügl|Frhr|ggfs|insb|autom|Mw[sS]t)\.[\u00A0\s]{1,2}</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>[\.!?…][\u0002|'|"|“|«|‹|\)|\]|\}¹²³]?[\u00A0\s]+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"“\p{Pe}\u00BB\u201D]?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\u00A0\s]\p{L}[\.!?…][\u00A0\s]{1,2}</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-<!-- z.B. 2 sentences: “Liebst du mich?” “Ja!” -->-<rule break="yes">-<beforebreak>[\.!?][”“]</beforebreak>-<afterbreak>[“„]</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Danish">-<!-- Split e.g.: He won't. Really. -->-<rule break="yes">-<beforebreak>'\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<!-- Split e.g.: He won't say no. Not really. -->-<rule break="yes">-<beforebreak>\sno[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<!-- Split at "a.m." or "p.m." followed by a capital letter. -->-<rule break="yes">-<beforebreak>[ap]\.m\.\s</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<!-- Don't split at e.g. "U. S. A." -->-<rule break="no">-<beforebreak>[^-\p{L}]\p{L}[\.!?…]['|"|«|\)|\]|\}]?\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't split at e.g. "U.S.A." -->-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't split after a white-space followed by a single letter followed- by a dot followed by another whitespace. e.g. " p. " -->-<rule break="no">-<beforebreak>\s\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!-- Don't split at "bla bla... yada yada" -->-<rule break="no">-<beforebreak>[\[\(]?\.\.\.[\]\)]?\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Don't split [.?!] when they're quoted -->-<rule break="no">-<beforebreak>['"][\.!?…]['"]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't break after quote unless there's a capital letter- e.g.: "That's right!" he said. -->-<rule break="no">-<beforebreak>["']\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Numbers, dates e.g. "3.10. datiert" -->-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<!-- f.eks. "Dette er ikke en(!) sætning." -->-<rule break="no">-<beforebreak>[\(\[][!?]{1,3}[\]\)]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- f.eks. "Dette er (nøjagtig!) en sætning." -->-<rule break="no">-<beforebreak>[!?]{1,3}[\)\]]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- f.eks. "bla (...) blubb" -> ingen sætning -->-<rule break="no">-<beforebreak>[\(\)\[\]]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- don't split at cases like "Fra den 13. til 14. i måneden." -->-<rule break="no">-<beforebreak>\d+\.\s</beforebreak>-<afterbreak>(og|eller|til)\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{N}+\.\s</beforebreak>-<afterbreak>januar|februar|marts|april|maj|ju[ln]i|august|september|oktober|november|december</afterbreak>-</rule>-<!-- einige deutsche Monate, vor denen eine Zahl erscheinen kann,- ohne dass eine Satzgrenze erkannt wird- (z.B. "am 13. Dezember" -> keine Satzgrenze) -->-<rule break="no">-<beforebreak>\d+\.\s</beforebreak>-<afterbreak>januar|februar|marts|april|maj|ju[ln]i|august|september|oktober|november|december</afterbreak>-</rule>-<!-- English abbreviations - but these work globally for all languages -->-<rule break="no">-<beforebreak>\b(Mrs?|No|pp|St|no|Sr|Jr|Bros|vs|esp|[Ff]ig|Jan|Feb|Mar|Apr|Ju[nl]|Aug|Sep|Sept|Oct|Okt|Nov|Dec|PhD|al|cf|Inc|Ms|Gen|Sen|Prof|Corp|Co)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bP[Hh]\.\s?</beforebreak>-<afterbreak>D\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.\s?</beforebreak>-<afterbreak>Eng\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bLL\.\s?</beforebreak>-<afterbreak>[BM]\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.\s?</beforebreak>-<afterbreak>Sc\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.\s?</beforebreak>-<afterbreak>Comp?\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.\s?</beforebreak>-<afterbreak>Arch\.?</afterbreak>-</rule>-<!-- Danish abbreviations - Word Boundary \b abbreviation dot \. -->-<rule break="no">-<beforebreak>\b(abs|abstr|adj|adm|adr|adv|afd|afg|afl|afs|afvig|agro|akad|akk|allr|alm|amer|anat|ang|anm|anv|apot|appos|apr|arab|arb|arkais|arkæol|arp|arr|art|ass|astr|att|attrib|aud|aug|aut)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(bag|barb|barnespr|bd|bdt|beg|besl|best|bet|bhk|biavl|bibet|bibl|bibliot|billard|billedl|biol|bjergv|bk|bl|bogb|bogh|bogtr|bornh|bot|br|bryg|bto|bygn|bødk)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ca|cand|Chr|cirk|cit|co|d|da|dagl|dans|dat|dec|def|demonstr|dep|dial|diam|dim|dipl|disp|distr|distrib|dobb|dok|dr|dvs|e|egl|ejd|eks|eksam|ekskl|eksp|ekspl|el|ell|ellipt|emb|endv|eng|enk|ent|etnogr|eufem|eur|event|evt)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(f|fagl|fakt|farv|feb|ff|fhv|fig|filos|fin|fisk|fk|fl|flg|flt|flyv|fmd|fon|foragt|forb|foreg|forf|forsikr|fors|forsk|forst|forv|foræld|fot|fr|fre|fris|frk|fsv|fuldm|fx|fys|fysiol|fægt|gart|gartn|garv|gdr|gen|genopt|geogr|geol|geom|germ|gl|glarm|glda|gldgs|glholl|glno|gns|got|gr|gradbøjn|graf|gram|gross|grundbet|græc|guldsm|gym)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(hat|hd?|hebr|henh|hensobj|herald|hhv|hist|hj|holl|hovedbet|hr|hty|højtid|haandarb|haandv|if?|iflg|ifm|ift|iht|imp|incl|indb|indik|inf|ing|Inkl|inkl|insp|instr|interj|intk|intr|iron|isl|ital|jan|jarg|jernb|jf|jnr|jr|jul|jun|jur|jy|jæg|jærnb|jød)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Kbh|kbh|kem|kgl|kirk|kl|kld|knsp|kog|koll|komm|komp|konj|konkr|kons|Kr|kr|kurv|kvt|køkkenspr|l|landbr|landmaaling|lat|lb|lic|lign|litt|Ll|log|Loll|loll|lrs|lør)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(m|maj|maks|mal|man|mar|mark|mat|mdl|mdr|med|medl|meng|merc|meteorol|meton|metr|mf|mfl|mht|mia|mil|min|mineral|mio|ml|mlat|mm|mnt|mods|modsætn|modt|mr|mrk|mur|mvh|mytol|møl|mønt|n|naturv|ndf|Ndr|nedsæt|nht|no|nom|nov|nr|nt|num|nyda|nydann|nylat|naal)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(obj|obl|oblik|obs|odont|oecon|oeng|ofl|ogs|oht|okt|oldfr|oldfris|oldn|olgn|omg|omkr|omtr|ons|opr|ordspr|org|osax|ovenst|overf|overs|ovf|p|pag|parl|part|pass|pat|pct|perf|pers|pga|ph|pharm|phil|pk|pkt|pl|plur|poet|pol|polit|pop|port|poss|post|pott|pr|prod|pron|propr|prov|præd|præp|præs|præt|psych|pt|pæd|paavirkn)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(reb|ref|refl|regn|rekl|relat|relig|resp|retor|rev|rid|rigsspr|run|russ|s|sa|sanskr|scient|sdjy|sdr|sek|sen|sep|sept|shetl|sj|sjæll|skibsbygn|sko|skol|skr|skriftspr|skræd|Skt|slagt|slutn|smed|sml|smsat|smst|snedk|soc|soldat|sp|spec|sport|spot|spr|sprogv|spøg|ssg|ssgr|st|stat|stk|str|straf|stud|subj|subst|superl|sv|sætn|søfors|søn)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(talem|talespr|tandl|td|tdl|teat|techn|telef|telegr|telekom|teol|th|theol|tir|tirs|tlf|told|tor|tors|trans|tsk|ty|tyrk|tøm|u|ubesl|ubest|udd|uddan|udenl|udg|udtr|uegl|ugtl|ult|underbet|undt|univ|upers|ur|urnord)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(vs?|var|varem|vbs|vedk|vedl|vedr|vejl|verb|vet|vha|vol|vsa|vulg|væv|zool|æ|æda|ænht|ænyd|æstet|ø|økon|å|årg|årh)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Danish abbreviations - These can appear at the end of sentence, so watch out for the next letter: it shouldn't be uppercase. -->-<rule break="no">-<beforebreak>\b(etc|mv|osv)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>[\.!?…][\u0002|'|"|«|\)|\]|\}¹²³]?\s+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\p{Pe}\u00BB\u201D]?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Esperanto">-<!-- Esperanto abbreviations (see http://eo.lernu.net/lernado/gramatiko/demandoj/mallongigoj.php) -->-<rule break="no">-<beforebreak>\b(div|[Ee]kz|h|[Ii]nkl|p|[Rr]ed|[Rr]im|ktp)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Bb]v|[Ĉĉ]|i\.a|k\.[acs]|[Tt]\.[ne]|k\.t\.p|n\.b|P\.S)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ll]ernu!\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>[\.!?…][\u0002|'|"|«|\)|\]|\}¹²³]?\s+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\p{Pe}\u00BB\u201D]?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="French">-<rule break="no">-<beforebreak>[\s\u00A0]</beforebreak>-<afterbreak>[»”’"'›]</afterbreak>-</rule>-<rule break="no">-<beforebreak>ambass|cuil|p|liv|assoc|bibl|ENREG|al|phot|circ|concl|deb|dest|dupl|éd|écon|incl?|ital|jur|juris|jurispr|larg|lex|législ|longit|(?-i)RR|(?-i)ÉÉm|(?-i)EExc|métr|méd|néol|obs|plur|préf|prog|publ|trib|trim|suiv|(?-i)LL|env|élem|ér|ét|hon|hypexp|conj|coop|ch|alph|anglic|app|pr|collab|paragr|sect|para|commiss|coord|dép|dir|gér|secour|sén|gén|abrév|adj|adr|anon|append|av|auj|bibl|bibliogr|bdc|boul|bull|bur|caar|cat|cell|chap|cir|compl|cf|corres|dest|dict|div|dom|dr|édif|éd|électr|élém|encycl|fig|fl|graph|hist|hyp|ill|imm|imp|impr|incl|inc|ind|in[gtvf]|jur|lat|litt|liq|loc|liv|livr|méd|mém|pl|réd|rel|sc|suiv|sup|suppl|trad|univ|mus|pharm|soc|pol|compt|urb|act|confect|exp|réal|prov|introd|inv|tial|enr|ép|équiv|esp|étym|excl|exc|ap|arr|arch|adv|al|anc|angl|ann|gest|gouv|prés|rect|représ|resp|scrut|vol|coll|réf|id|sqq?|janv|fév|avr|juill|oct|nov|déc|admin</beforebreak>-<afterbreak>\p{Ll}.*</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Ll}.*</beforebreak>-<afterbreak>ambass|cuil|p|liv|assoc|bibl|oct|déc|jan|fév|avr|juil|sept|nov|ENREG|al|circ|concl|deb|dest|dupl|éd|écon|incl?|ital|jur|juris|jurispr|larg|lex|législ|longit|(?-i)RR|(?-i)ÉÉm|(?-i)EExc|métr|méd|néol|obs|plur|préf|prog|publ|trib|trim|suiv|(?-i)LL|env|élem|ér|ét|hon|hypexp|conj|coop|ch|alph|anglic|app|pr|collab|paragr|sect|para|commiss|coord|dép|dir|gér|secour|sén|gén|abrév|adj|adr|anon|append|av|auj|bibl|bibliogr|bdc|boul|bull|bur|caar|cat|cell|chap|cir|compl|cf|corres|dest|dict|div|dom|dr|édif|éd|électr|élém|encycl|fig|fl|graph|hist|hyp|ill|imm|imp|impr|incl|inc|ind|in[gtvf]|jur|lat|litt|liq|loc|liv|livr|méd|mém|pl|réd|rel|sc|suiv|sup|suppl|trad|univ|mus|pharm|soc|pol|compt|urb|act|confect|exp|réal|prov|introd|inv|tial|enr|ép|équiv|esp|étym|excl|exc|ap|arr|arch|adv|al|anc|angl|ann|gest|gouv|prés|rect|représ|resp|scrut|vol|coll|réf|id|sqq?|janv|fév|avr|juill|oct|nov|déc|admin</afterbreak>-</rule>-<rule break="no">-<beforebreak>.*°C</beforebreak>-<afterbreak>de</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?][\s\u00A0][»”’"'›][\s\u00A0]</beforebreak>-<afterbreak>[«“‘‹"'\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>Yahoo![\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- !? + lowercase -->-<rule break="no">-<beforebreak>(\!|\?)[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\.\[\d+\][\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(https?|ftp|file|chrome|chromium|android|(chrome|moz)\-extension):///?[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+\.(fr|com|net|org|info|de|es|edu|co|eu|nl|io|cn|uk|gov|biz|ca|tk|ru|br|jp|pl)(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z]{2,5}(\.|\b)</afterbreak>-</rule>-<!-- French abbreviations -->-<rule break="no">-<beforebreak>\b((?iu)J\.\-C|art|app|cf|chap|e(nv|tc)|fém|fig|masc|p|sing|suiv|suppl|tél|op|ex)\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(etc)\.\)[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(apr|ave?|boul|Mr?|Mrs|MM?|Mlle)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Ps}[!?]+\p{Pe} </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.!?…]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>["”'’][\s\u00A0]*</beforebreak>-<afterbreak>[\s\u00A0]*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”][\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.[\s\u00A0]</beforebreak>-<afterbreak>\p{L}\.[\s\u00A0]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>(…|\.\.\.)[\s\u00A0]?\)[\s\u00A0]</beforebreak>-<afterbreak>[^\p{P}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>(…|\.\.\.)[\s\u00A0]?\?\)[\s\u00A0]</beforebreak>-<afterbreak>[^\p{P}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Lu}\p{L}+[\s\u00A0]v\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Lu}\p{L}+</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[^,][\s\u00A0]\p{L}{2}\.[\s\u00A0]</beforebreak>-<afterbreak>\p{N}+\)[\s\u00A0]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s\u00A0]\p{L}{1,2}\.[\s\u00A0]</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.[\s\u00A0]\p{Lu}\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\p{Lu}\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[^\.][\s\u00A0][A-Z]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(:?Blvd|Ave|Mts?)\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(?:Kan|Ill|M[ai]ss)\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\(\p{Ll}+\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>i\.e\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[A-Z]\.[A-Z]\.</beforebreak>-<afterbreak>[A-Z]\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bL\.</beforebreak>-<afterbreak>A\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bU\.</beforebreak>-<afterbreak>[SK]\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[nN]o\.[\s\u00A0]</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bP[Hh]\.[\s\u00A0]?</beforebreak>-<afterbreak>D\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\be\.g\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvs\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--"Etc." can end the sentence, so we check for the uppercase letter after it.-->-<rule break="no">-<beforebreak>\b[Ee]tc\.[\s\u00A0]</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Bb]tw|BTW)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>(?i)FRITZ!</beforebreak>-<afterbreak>(?i)Box</afterbreak>-</rule>-<rule break="no">-<beforebreak>ID.</beforebreak>-<afterbreak>3|4|Buzz|Crozz</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bP[Hh]\.?[\s\u00A0]?[Dd]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(P[hH][dD]|BSc|BEng|BComp|BArch|MSc|MEng|MComp)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bLL\.[\s\u00A0]?[BM]\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Eng\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bLL\.[\s\u00A0]?</beforebreak>-<afterbreak>[BM]\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Sc\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Comp?\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.[\s\u00A0]?</beforebreak>-<afterbreak>Arch\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[BM]\.?[\s\u00A0]?(Sc|Eng|Comp|Arch)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bI(nc|NC)\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bCorp\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bBros\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bLtd\.[\s\u00A0]</beforebreak>-<afterbreak>\p{Ll}+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bCo\.[\s\u00A0]</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bE\.[\s\u00A0]</beforebreak>-<afterbreak>\b[Cc]oli\b</afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>[\.!?…][\u0002|'|"|«|\)|\]|\}¹²³]?[\s\u00A0]+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\p{Pe}\u00BB\u201D]?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\s\u00A0]\p{L}[\.!?…][\s\u00A0]</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Ukrainian">-<!-- when sentence starts with ellipsis: ...Мазій і Юхим теж. -->-<rule break="no">-<beforebreak>(^|[\h])(\.\.\.|…)</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<!-- Наші в... Лос-Анджелесі -->-<rule break="no">-<beforebreak>\b(в|у|на|за|з|із|зі|зо)(\.\.\.|…)[\h\v]*</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[.!?…][»“]?[\h]+</beforebreak>-<afterbreak>[\h]*([«"„“(]|[‐-―-][\h])\p{Ll}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\v[\h]*</beforebreak>-<afterbreak>(\.\.\.|…)</afterbreak>-</rule>-<!-- Digit as a point number: 1. перший пункт -->-<rule break="no">-<beforebreak>\b\d{1,3}\.[\h]+</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<!-- various punctuation between lowercase letters -->-<rule break="no">-<beforebreak>\b\p{Ll}+[.!?][\h\v]*</beforebreak>-<afterbreak>\h*(([\(«]|[\[‐-―-][\h\v]*)?\p{Ll})</afterbreak>-</rule>-<rule break="no">-<beforebreak>([\[\(]*[\]\)]*|\.\.\.|…)[\h\v]+</beforebreak>-<afterbreak>[\h\v]*\p{Ll}</afterbreak>-</rule>-<!-- lowercase letter abbreviations together: н.е., кв.м. -->-<rule break="no">-<beforebreak>\b\p{L}{1,2}\.</beforebreak>-<afterbreak>\p{L}{1,2}\.</afterbreak>-</rule>-<!-- latin capital char abbreviations A. B. C. -->-<rule break="no">-<beforebreak>\b[\u00A0\u202F]?[A-Z]\.[\h\v]?</beforebreak>-<afterbreak>[A-Z][a-zA-Z'’.-]|[А-ЯІЇЄҐ]\.</afterbreak>-</rule>-<!-- capital char abbreviations А. Б. В. -->-<rule break="no">-<beforebreak>(^[\h\v]*|\([\h\v]*|[«„"]|(\b[А-ЯІЇЄҐACEIHOPX]\.-))[А-ЯІЇЄҐA-Z]\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Іван Ч. (1914 р. н.) -->-<rule break="no">-<beforebreak>[\h\v][А-ЯІЇЄҐ]\.[\h\v]*</beforebreak>-<afterbreak>[А-ЯІЇЄҐ]\.|[0-9]|[\h\v]*,|[\h\v]*[:«]|\([0-9]{4}</afterbreak>-</rule>-<!-- І. Коваль -->-<rule break="no">-<beforebreak>[\h\v.]([А-ЯІЇЄҐACEIHOPX]\.-)?(?<!°)[А-ЯІЇЄҐABCEIHOPX](?<!(Куан[\h]+Ю|(Петр|Олександр)([аоу]|ові|ом)?[\h]+[IІ]+))\.[\h\v]*</beforebreak>-<afterbreak>[А-ЯІЇЄҐ][а-яіїєґА-ЯІЇЄҐ'’ʼ]{3}</afterbreak>-</rule>-<!-- Ів. Франко (але Ів Бутільє) -->-<rule break="no">-<beforebreak>(^|[\h\v])(Ів|Дж)\.[\h\v]+</beforebreak>-<afterbreak>[А-ЯІЇЄҐA-Z]</afterbreak>-</rule>-<!-- Year: 2000 р.:- виробила у 2018 р. 8,1 млн декалітрів- від 26 квітня 2017 р. №35- а до лютого 2020 р. — затвердити--->-<rule break="no">-<beforebreak>\b([0-9]{2}|[0-9]{4})[\h\v]+р\.[\h\v]+</beforebreak>-<afterbreak>[\h\v]*[№0-9‐-―-]</afterbreak>-</rule>-<!-- річка - р. Дніпро -->-<rule break="no">-<beforebreak>(?<!\d[\h]*)\bр\.[\h\v]*</beforebreak>-<afterbreak>[\h]*(?!(На|Але|Так?)[\h\v]+)[А-ЯІЇЄҐA-Z][^\h]</afterbreak>-</rule>-<!-- У травні 1949 р. Грушківський район -->-<rule break="no">-<beforebreak>[А-ЯІЇЄҐ][а-яіїєґ'’-]*([\h]+[а-яіїєґ'’-]+)?[\h](\d{4}[‐-―-])*\d{4}[\h]*р\.[\h\v]*</beforebreak>-<afterbreak>[\v\h]*(?!(На|Але|Так?)[\h\v]+)[А-ЯІЇЄҐA-Z][^\h\v]</afterbreak>-</rule>-<!-- 15 вересня 1995 р. Україною було підписно -->-<rule break="no">-<beforebreak>\d{1,2}[\h]+[а-яіїєґ]+[\h]\d{4}[\h]*р\.[\h\v]*</beforebreak>-<afterbreak>[\h]*(?!(На|Але|Так?)[\h\v]+)[А-ЯІЇЄҐA-Z][^\h\v]</afterbreak>-</rule>-<!-- Years: рр. -->-<rule break="no">-<beforebreak>\b([0-9]0|[0-9]{3}0)(-[мх])?рр\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- млн./млрд./грн. — frequent mistake -->-<rule break="no">-<beforebreak>\b(тис|млн|млрд|грн)\.[\h\v]*</beforebreak>-<afterbreak>[\h\v]*(\d|[КМ]Вт)</afterbreak>-</rule>-<!-- усталені скорочення, що не збігаються з нескороченими словами -->-<rule break="no">-<beforebreak>\b(укр|рос|англ?|амер|італ|ісп|нім|фр(анц)?|лат|грец(ьк)?)\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(абз|арк|ауд|бл|буд|бульв|вул|держ|дод|зав|зб|зв|зовн|екон|к|кв|канд|кн|напр|нпр|нац|обл|оп|пл|пол|поч|пп|пор|просп|розд|стор|табл|[Тт]]ел|ч|част)\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(кін)\.[\h\v]*</beforebreak>-<afterbreak>[а-яіїєґ0-9IXV]|[ІХ]+\b</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[сС]т\.[\h\v]</beforebreak>-<afterbreak>[\h]*(?!([АВУОІЄ]|На|Але|Так?)[\h\v])</afterbreak>-</rule>-<!-- нар. 1945 р. | (1966 р. нар.) | 1975 — нар. Осипчук -->-<rule break="no">-<beforebreak>([0-9]|[-–—])[\h\v]+нар\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bнар\.[\h\v]*</beforebreak>-<afterbreak>([0-9]|бл\.|арт\.)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bдол\.[\h\v]*</beforebreak>-<afterbreak>США</afterbreak>-</rule>-<!-- п. 10 від 11.10.1933, д. Василь -->-<rule break="no">-<beforebreak>(?<!т\.[\h\v]?)\b[пд]\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- усталені скорочення, що збігаються з нескороченими словами -->-<rule break="no">-<beforebreak>\b(див)\.[\h\v]</beforebreak>-<afterbreak>[\h\v]*[^А-ЯІЇЄҐ]</afterbreak>-</rule>-<!-- Верховний орган, див. Африканський національний конгрес -->-<rule break="no">-<beforebreak>[,‐-―-][\h\v]*(див)\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- скорочення в дужках:- України (див. Зимові походи)--->-<rule break="no">-<beforebreak>(\([^)]*|\[[^\]]*|,[\h\v]*)\b(див)\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- abbreviation with proper noun: проф. Грицько, о. Лісове -->-<rule break="no">-<beforebreak>\b(ап|[Аа]кад|[Пп]роф|[Дд]оц|[Аа]сист|[Рр]еж|[Аа]рх|[Сс]вв?|о|оз|ім|інж|дир|тов|упоряд|тт|чл\.-кор|[Пп]реп)\.[\h\v]*</beforebreak>-<afterbreak>[\h\v]*[А-ЯІЇЄҐA-Z]</afterbreak>-</rule>-<rule break="no">-<beforebreak>(?<![іи]\s+)\bдр\.[\h\v]*</beforebreak>-<afterbreak>[\h\v]*[А-ЯІЇЄҐ]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bМан\.[\h\v]*</beforebreak>-<afterbreak>[\h\v]*([Сс]іті|[Юю]н)</afterbreak>-</rule>-<!-- смерть гр. Болтаровича, but not "9 гр." -->-<rule break="no">-<beforebreak>[^0-9][\h\v]+[Гг]р\.[\h\v]*</beforebreak>-<afterbreak>[\h\v]*[А-ЯІЇЄҐA-Z]</afterbreak>-</rule>-<!-- TODO: арт. - артист -->-<rule break="no">-<beforebreak>\b([Аа]рт|[Мм]ал|[Рр]ис)\.[\h\v]*</beforebreak>-<afterbreak>[\h\v]*[0-9]</afterbreak>-</rule>-<!-- ХІІ р., 3-6 арт., 2-3 тт. -->-<rule break="no">-<beforebreak>[0-9][\h\v]+(арт|тт)\.[\h\v]*</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- але розбиваємо «всього 20 м. Почалося» -->-<rule break="no">-<beforebreak>(?<!\d[\h\v]*)\bм\.[\h\v]*</beforebreak>-<afterbreak>[А-ЯІЇЄҐ][а-яіїєґ']</afterbreak>-</rule>-<!-- село/сторінка/місто, але щоб не збігалося з секундами/метрами -->-<rule break="no">-<beforebreak>([\h\v][«(][см]|[^0-9/. ][\h\v]+[см])\.[\h\v]+</beforebreak>-<afterbreak>[А-ЯІЇЄҐ][а-яіїєґ']</afterbreak>-</rule>-<!-- (реж. Емманюель -->-<rule break="no">-<beforebreak>[(«"„“”\[‹][а-яіїєґ]+\.[\h\v]+</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- оренда кабінетів коштувала (!) 22,36 млн грн. -->-<rule break="no">-<beforebreak>["«„“”\[][.!?…]{1,3}["»”“\]][\h\v]+</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- статус правових держав. — Авт.). -->-<rule break="no">-<beforebreak></beforebreak>-<afterbreak>[\h\v]*[‐-―-][\h\v]*([Рр]ед|[Аа]вт)[\h\v]*\.[\)\]]</afterbreak>-</rule>-<!-- Цензор.НЕТ -->-<rule break="no">-<beforebreak>[а-яіїєґ]\.</beforebreak>-<afterbreak>НЕТ|Інфо|Info|City|Life|UA|Ру</afterbreak>-</rule>-<!-- але лишаємо ініціали: С.\u202F Шелухин -->-<rule break="yes">-<beforebreak>(?<!\h[А-ЯІЇЄҐ])[.!?…]{1,3}\u202F[\h\v]+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[.!?…]['»"„“”\]›\u0002]*[\h\v]+</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Всього 33 тис.А можей й більше -->-<rule break="yes">-<beforebreak>[.!?…]['»"„“”)\]›]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<!-- “Слон” (2008 р.) У минулому харків’янка -->-<rule break="yes">-<beforebreak>[.!?…]['»"„“”)\]›]?[\h\v]+</beforebreak>-<afterbreak>([‐-―-][\h\v]*)?\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Belarusian">-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<!-- capital char abbreviations А. Б. В. -->-<rule break="no">-<beforebreak>\b[А-ЯЁ]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bЎ\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Z]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!-- date/time -->-<rule break="no">-<beforebreak>\b[0-9]+(г)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[XVILMC]+(ст)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-9]+(\.|:)[0-9][0-9]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[0-9]+(\.|:)[0-9][0-9](\.|:)[0-9][0-9]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Measures -->-<rule break="no">-<beforebreak>\b[0-9]+(г|гг|грн|млн|млрд|руб|тыс)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- other abbreviations -->-<rule break="no">-<beforebreak>\b(в|вв|г|гг|грн|млн|млрд|руб|ст|р|тыс)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>['"„][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\u00AB][\.!?…][\u00BB]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>["”'\u00BB]\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<!-- break -->-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\u005D\u005D\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['»"”\p{Pe}]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Galician">-<!-- s. XIX; s.IX; sec. XX; séc. XX -->-<rule break="no">-<beforebreak>\bs([eé]c)?\.\s?</beforebreak>-<afterbreak>[IVXVDMCL]+</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[Ee]tc\.[\p{Pe}\p{Pf}\p{Pd}"']*\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(m[aá]x|m[ií]n|[aA]prox)\.[\p{Pe}\p{Pf}\p{Pd}"']*\s</beforebreak>-<afterbreak>[\p{Ll}\p{N}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([aA]pt?do|[aA]sdo|[aA]vd?a?|[Cc]ód|[Dd]e?pto|[Ff]ac|[Ii]nst)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- S.A.R. Felipe de Borbón -->-<rule break="no">-<beforebreak>\b(S\.A\.[IRS]\.|S\.R\.M\.|A\.R\.|S\.[ME]\.)\s</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([\p{Ll}\p{Lu}]\.)+[\p{Pe}\p{Pf}\p{Pd}"']*\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- S.A., s. l. -->-<rule break="no">-<beforebreak>\b\p{L}\.\s?</beforebreak>-<afterbreak>((\p{L}\.\s?)+|\p{Ll})</afterbreak>-</rule>-<!-- EE.UU; EE.UU.; EE. UU.; EE. UU -->-<rule break="no">-<beforebreak>\b\p{Lu}{2}\.\s?</beforebreak>-<afterbreak>(\p{Lu}{2}\.?\s?|\p{Ll})</afterbreak>-</rule>-<!-- tratamento // pendente: "PP. Escolapios" vs. "do PP. Bla bla" -->-<rule break="no">-<beforebreak>\b([Aa]fm[oa]s?|Emcia|Ilt?m[ao]s?|Iltres?|MM|Exc?m[ao]s?|Magf[oa]|D(na)?|Sra?|Sr[ea]s|Srta|Dra?|Dr[ea]s?|Rm[ao]|Rev|Revm[ao]|Mons|Emmo|Rv?d[ao]|[Ll]icd[oa]|[Ll]ic|[Ll]d[oa]|[pP]rofs?)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- num; figs; ca -->-<rule break="no">-<beforebreak>\b([Nn][úu]ms?|[fF]igs?|[Pp][aá]xs?|pp?|cc?a)\.\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<!-- num; figs -->-<rule break="no">-<beforebreak>\b([Vv]ols?|[Cc]aps?)\.\s</beforebreak>-<afterbreak>\p{N}|[IVXVDMCL]+</afterbreak>-</rule>-<!-- Admóns, admtva, Advo, Admtvos -->-<rule break="no">-<beforebreak>\b([Aa]dmóns?|[aA]d(mt)?v[oa]s?)\.\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- abrevs, Abl, adx, Acr, conx, loc, prep, subst -->-<rule break="no">-<beforebreak>\b([pP]pal|[Vv]des?|[Ii]b[íi]d|[Rr]efs?|[Cc]it|[Aa]brevs?|[Aa]bl|[Aa]dx|[Aa]dv|[Aa]cr|[Cc]onx|[Ll]oc|[Pp]rep|[Ss]ubst)\.\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([vV]id|[Cc]fr?)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- coords.; Ed.; eds.; Coord. -->-<rule break="no">-<beforebreak>\b([Cc]oord|[Ee]d)s?\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b([Cc]t[ae]|[Tt]e?lf|[Tt]fno|[Tt]el[eé]f|[Tt]el)s?\.\s</beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<!-- multielemento -->-<rule break="no">-<beforebreak>\b([oO]p|[lL]oc)\.\s?</beforebreak>-<afterbreak>cit\.</afterbreak>-</rule>-<!-- et al. -->-<rule break="no">-<beforebreak>\bet [aá]l\.\s?</beforebreak>-<afterbreak>[\p{Ll}\p{N}]</afterbreak>-</rule>-<!-- Don't split [.?!] when they're quoted -->-<rule break="no">-<beforebreak>['"\(][\.!?…]['"\)]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bvs\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\besp\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>(…|\.\.\.)\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- narrator comments in dialogs -->-<rule break="no">-<beforebreak>[\.!?…]\s</beforebreak>-<afterbreak>[-–—],?\s\p{Ll}.+</afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>[\.…][\u00BB\u2019\u201D\u203A"'\u0002]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}+[\p{Pf}\p{Pe}\u00BB\u2019\u201D\u203A"'\u0002]*[\.:!?…]+\s*</beforebreak>-<afterbreak>[¡¿«»"'\p{Ps}]*\p{Lu}\p{L}*</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[^\s]:\s</beforebreak>-<afterbreak>['"«¡¿\p{Ps}\p{Pi}]?\p{Lu}\p{Ll}*</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Ideographic">-<rule break="no">-<beforebreak>[:]+[\p{Pe}\p{Pf}\p{Po}"-[\u002C\u003A\u003B\u055D\u060C\u061B\u0703\u0704\u0705\u0706\u0707\u0708\u0709\u07F8\u1363\u1364\u1365\u1366\u1802\u1804\u1808\u204F\u205D\u3001\uA60D\uFE10\uFE11\uFE13\uFE14\uFE50\uFE51\uFE54\uFE55\uFF0C\uFF1A\uFF1B\uFF64]]*</beforebreak>-<afterbreak>\s+\P{Lu}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[:]+[\p{Pe}\p{Pf}\p{Po}"-[\u002C\u003A\u003B\u055D\u060C\u061B\u0703\u0704\u0705\u0706\u0707\u0708\u0709\u07F8\u1363\u1364\u1365\u1366\u1802\u1804\u1808\u204F\u205D\u3001\uA60D\uFE10\uFE11\uFE13\uFE14\uFE50\uFE51\uFE54\uFE55\uFF0C\uFF1A\uFF1B\uFF64]]*</beforebreak>-<afterbreak>\s</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[。.!?…]+</beforebreak>-<afterbreak>.</afterbreak>-</rule>-<rule break="no">-<beforebreak>\.\.\.</beforebreak>-<afterbreak>\s+\P{Lu}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>^\s*\p{Nd}+[\p{Nd}\.\)\]]+\s+</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.\?\!]+</beforebreak>-<afterbreak>\s</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Breton">-<rule break="no">-<beforebreak>\b[dD]\.l\.e\.?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"”»\p{Pe}\u00BB\u201D]?\s*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>[\.!?…][\u0002|'|"|«|\)|\]|\}¹²³]?\s+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\p{Pe}\u00BB\u201D]?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Portuguese">-<rule break="no">-<beforebreak>\b(https?|ftp|file|chrome|chromium|android|(chrome|moz)\-extension):///?[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+(\.|\b)</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b[A-Za-z0-9\-]+\.</beforebreak>-<afterbreak>[A-Za-z0-9\-]+\.(com|net|org|info|de|es|edu|co|eu|nl|io|cn|uk|gov|biz|ca|tk|ru|br|jp|pl)(\.|\b)</afterbreak>-</rule>-<!-- Abbreviations that cannot finish sentences-->-<rule break="no">-<beforebreak>\b(a|Ab|abr|abrev|absol|acad|Açor|A\. ?D|add|adj|adv|advers|Aeron|afér|Agric|ago|Álg|aprox|[Aa]rts?|Artilh|auxil|av|Av)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Bot|barb|B\.el|Bibl|Biol|Bioquím|burl)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ca|card|cat|caus|cf|cit|cód|comp|compar|conj|contr|coord|cop)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(D|def|dem|deprec|deriv|det|dez|disj|[Dd]ra?s?)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Ecol|Econ|ed|elem|Eng|erud|estrang|ex|Ex)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(etc)\.\s?</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(f|fam|Farm|fem|fev|fig|fin|fl|fr|frac)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(gén|geog|Geogr|Geol|Geom|gír|gloss|Gram)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(hab|hist|Hort)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Ibid|id|i.e|incompat|indef|inf|infant|Inform|integr|interj|interr|intr|inv)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(jan|jul|jun|Jorn|Jur)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(lat|Lat|Lda|Ling|Lit|liv|loc|log|Lóg|long)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(m|mai|mar|masc|Mat|máx|Mecân|[Mm]ed|Mil|mín|mult|Mús)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(n|N|Náut|N.B|neg|neol|nov|num|núm)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ord|out)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(pág|págs|Paleont|part|pass|[Pp]edag|pejor|pess|Pesc|p|Pe|p.f|pl|pleb|p.m|poét|[Pp]olít|pop|pov|poss|p.p|p.p.m|pp|pref|prep|[Pp]rof|pron|P.S)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(q.b|q.do|Q.E|Q.I|ql)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(R|rel|Relig|Rev)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(S|S.A|set|símb|S. ?M|[Ss]ra?s?|[Ss]rta|suf|superl)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(t|tip|Tip|tít|top|[Tt]opogr|tr|trad|Trás-os-M|trim)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Univ)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(v|V|vd|vid|voc|vol|V.S|vs|vulg)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Zool)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- s. XIX; s.IX; sec. XX; séc. XX -->-<rule break="no">-<beforebreak>\bs([eé]c)?\.\s?</beforebreak>-<afterbreak>[IVXDMCL]+</afterbreak>-</rule>-<!-- English abbreviations - but these work globally for all languages -->-<rule break="no">-<beforebreak>\b(Mr|Mrs|No|pp|St|Jr|Bros|etc|vs|esp|[Ff]ig|PhD|al|cf|Inc|Ms|Gen|Sen|Prof|Corp|Co|Ltd)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Latin abbreviations - but these work globally for all languages -->-<rule break="no">-<beforebreak>\b(sp|spp)\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- initials: A. C. Jones. -->-<rule break="no">-<beforebreak>\b[A-ZÀÉÈÍÓÒÚ]\.\s?</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Two final stop abbreviations -->-<rule break="no">-<beforebreak>\b[ad]\.\s?</beforebreak>-<afterbreak>C\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>p\.\s\?</beforebreak>-<afterbreak>d\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>p\.\s\?</beforebreak>-<afterbreak>ex\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>P\.\s?</beforebreak>-<afterbreak>S\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>\bP[Hh]\.\s?</beforebreak>-<afterbreak>D\.?</afterbreak>-</rule>-<rule break="no">-<beforebreak>V\.\s?</beforebreak>-<afterbreak>Rev\.</afterbreak>-</rule>-<rule break="no">-<beforebreak>v\.\s?</beforebreak>-<afterbreak>g\.</afterbreak>-</rule>-<!-- Don't split after a white-space followed by a single letter followed- by a dot followed by another whitespace. e.g. " p. " -->-<rule break="no">-<beforebreak>\s\p{L}\.\s?</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!-- Any word in acronyms like E.U.A. or P.S.P. or C. or c.s.p. or p. e. -->-<rule break="no">-<beforebreak>\b(\p{L}\.)+[\p{Pe}\p{Pf}\p{Pd}"”']*\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Any word in acronyms like EE.UU. or BB. DD. -->-<rule break="no">-<beforebreak>\b([\p{Lu}]{2}\.)+[\p{Pe}\p{Pf}\p{Pd}"”']*\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- máx. mín. etc. -->-<rule break="no">-<beforebreak>\b([Ee]tc|m[aá]x|m[ií]n|aprox|\d+o)\.[\p{Pe}\p{Pf}\p{Pd}"”'’]*\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Composed abbrev. e.g. et al. -->-<rule break="no">-<beforebreak>\bet al\.[\p{Pe}\p{Pf}\p{Pd}"”']*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Units -->-<rule break="no">-<beforebreak>\b([Ee]sc|K[gm]s?|[mc]?[gml]s]|[Hh](rs)?)\.[\p{Pe}\p{Pf}\p{Pd}"”'’]*\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- Split at e.g. "1a. There is..." -->-<rule break="yes">-<beforebreak>\d+[a-z]\.\s?</beforebreak>-<afterbreak>\p{Lu}</afterbreak>-</rule>-<!-- Don't split at cases like "in 13. or 14. paragraphs" -->-<rule break="no">-<beforebreak>\d+\.\s?</beforebreak>-<afterbreak>(e|ou|até)\s</afterbreak>-</rule>-<!-- Don't split [.?!] when they're quoted -->-<rule break="no">-<beforebreak>['"“][\.!?…]['"”]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Not break for ellipses (...) -->-<rule break="no">-<beforebreak>[^\s](\Q...\E|…)\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- z.B. "bla (...) blubb" -> without ending sentence -->-<rule break="no">-<beforebreak>[\(\)\[\]]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't break after quote unless there's a capital letter- e.g.: "That's right!" he said. -->-<rule break="no">-<beforebreak>["”'’]\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- z.B. "This is a (!) exclamation." -->-<rule break="no">-<beforebreak>[\(\[][!?]{1,3}[\]\)]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- z.B. "This here is an (awesome!) phrase." -->-<rule break="no">-<beforebreak>[!?]{1,3}[\)\]]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- narrator comments in dialogs -->-<rule break="no">-<beforebreak>[\.!?…]\s</beforebreak>-<afterbreak>[-–—],?\s\p{Ll}.+</afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>\b(etc)\.\s?</beforebreak>-<afterbreak>\p{Lu}\p{Ll}*</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?][\u0002|'|"|“|«|\)|\]|\}¹²³]?\s+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"”»\p{Pe}\u00BB\u201D]?\s*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}*</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[^\s]:\s</beforebreak>-<afterbreak>['"«¡¿\p{Ps}\p{Pi}]\p{Lu}\p{Ll}*</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\r?\n</beforebreak>-<afterbreak>\p{Lu}\p{Ll}*</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Italian">-<!-- Italian abbreviations A - C-->-<rule break="no">-<beforebreak>\b(a\.c|a\.C|ad es|all|Amn|Arch|Avv|Bcc|Cav|c\.a|C\.A\.P|Cc|banc|post|c\.c\.p|c\.m|Co|c\.p|C\.P|C\.p\.r|corr|c\.s|c\.v)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Chia\.mo|C\.so|Circ\.ne)\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Italian abbreviations D - L -->-<rule break="no">-<beforebreak>\b(d\.C|Dott|Dr|ecc|Egr|e\.p\.c|fatt|FF\.AA|FF\.SS|Geom|Gen|g|gg|Id|Ing|int|lett)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Dott\.ssa|Egr\.i|Egr\.ia|F\.lli|Gent\.mo|Gent\.mi|Gent\.ma|Gent\.me|Ill\.mo|L\.go)\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Italian abbreviations M - P -->-<rule break="no">-<beforebreak>\b(Mo|Mons|N\.B|n|ogg|On|p|pag|par|pp|p\.c|p\.c\.c|p\.es|p\.f|p\.r|P\.S|p\.v|P\.T|Prof)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(P\.zza|P\.le|Preg\.mo|Prof\.ssa)\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Italian abbreviations R - S -->-<rule break="no">-<beforebreak>\b(R|racc|Rag|Rev|ric|Rif|R\.P|R\.S\.V\.P|S\.A|S\. acc|S\.B\.F|seg|sgg|ss|S|Ss|Sig|Sigg|s\.n\.c|Soc|S\.p\.A|Spett|S\.P\.M|S\.r\.l)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(Sig\.na|Sig\.ra|Stim\.mo)\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Italian abbreviations T - V -->-<rule break="no">-<beforebreak>\b(tel|u\.s|V|V\.P|v\.r|v\.s)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(V\.le)\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Italian dictionary abbreviations -->-<rule break="no">-<beforebreak>\b(abbr|acron|agg|art|avv|card|compar|conf|cong|det|dim|f|fonosimb|ger|impers|indef|indet|inter|intr|inv|lat|loc|m|n|num|ord|p|pers|pl|pass|pres|pref|prep|pron|ponom|rel|s|sost|simb|suff|ter|tr|v|var)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Tamil">-<rule break="no">-<beforebreak>\bஎ\.கா\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ஜன|பிப்|மார்|ஏப்|ஆக|செப்|அக்|நவ|டிச)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ரூ|ரி\.ம|பக்)\.\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(கி\.பி|கி\.மு)\.\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(ஐ\.நா|தி\.மு\.க|அ\.இ\.அ\.தி\.மு\.க|அ\.தி\.மு\.க|ம\.தி\.மு\.க|ம\.இ\.கா|இ\.ஆ\.ப|ஐ\.ஏ\.எஸ்|எம்\.பி|எம்\.எல்\.ஏ|எம்\.ஜி\.ஆர்|டி\.எம்\.எஸ்)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\bபி\.கு\.:?\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b(பி\.இ|பி\.ஏ|ஏம்\.பி\.பி\.எஸ்|பி\.ஏ\.பி\.எல்|எம்\.ஏ|எம்\.எஸ்\.சி|எம்\.இ|எம்.லிட்|பி\.எச்\.டி)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002\]\}¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Generic">-<rule break="yes">-<beforebreak>[\.!?…][\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Persian">-<rule break="no">-<beforebreak>\b(نه|بله)\!\s</beforebreak>-<afterbreak>\p{N}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Ps}[!?؟]+\p{Pe} </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.!?؟…]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[«»"”']\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[«'"„][\.!?؟…]['"”»]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[^,،][\s]\p{L}{2}\.\s</beforebreak>-<afterbreak>\p{N}+\)\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s]\p{L}{1,2}\.\s</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\s\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[^\.]\s[ضصثقفغعهخحجچشسیبلاتنمکگ\ظطزرذدپوًٌٍَُِّْA-Z]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\(\p{Ll}+\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?؟…][«»\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?؟…][«»'"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?؟…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Serbian">-<!-- Don't split at e.g. "d. h." -->-<rule break="no">-<beforebreak>[^-\p{L}'’]\p{L}[\.!?…]['|"|“|«|\)|\]|\}]?\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Не раздвајати код на пр. "U.S.A." -->-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't split after a white-space followed by a single letter followed- by a dot followed by another whitespace. e.g. " p. " -->-<rule break="no">-<beforebreak>\s\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!--Не раздвајати код "бла бла... трућ трућ".-->-<rule break="no">-<beforebreak>[\[\(]?\.\.\.[\]\)]?\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!--Не раздвајати [.?!] када се налазе унутар знакова навода,-једноструких или двоструких. -->-<rule break="no">-<beforebreak>['"„][\.!?…]['"“]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't break after quote unless there's a capital letter- e.g.: "That's right!" he said.--Не раздвајати после наводника осим ако нису праћени-великим словом. На пример:- "Тако је!", рече он.-->-<rule break="no">-<beforebreak>["'“],\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<!-- e.g. "Das ist . so." - assume one sentence. -->-<rule break="no">-<beforebreak>\s([\.!?]{1,3}|…)['|"|“|«|\)|\]|\}]?\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Бројеви, датуми на пр. "Дана 3.10."-->-<rule break="no">-<beforebreak>\b\d+\.\s</beforebreak>-<afterbreak>\p{Ll}|\p{Lu}{2,}</afterbreak>-</rule>-<!--На пр.: "Ово овде је такође(!) једна реченица".-->-<rule break="no">-<beforebreak>[\(\[][!?]{1,3}[\]\)]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--На пр.: "Ово овде је (такође!) једна реченица."-->-<rule break="no">-<beforebreak>[!?]{1,3}[\)\]]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Не раздвајај у случају као на пр.: "Петар I дошао је ..."-->-<rule break="no">-<beforebreak>[\s ][IVX]+\s</beforebreak>-<afterbreak>[^\p{Lu}]+</afterbreak>-</rule>-<!--Не раздвајај у случају као "од 13. до 14. века"-->-<rule break="no">-<beforebreak>\d+\.\s</beforebreak>-<afterbreak>(и|или|до)\s</afterbreak>-</rule>-<!--Не раздвајај у случају датума писаног мешовито - дан арапским цифрама,-а назив месеца словима: "Дне 28. јуна"-->-<rule break="no">-<beforebreak>\d+\.\s</beforebreak>-<afterbreak>јануар|јануара|фебруар|фебруара|март|марта|април|априла|мај|маја|јун|јуна|јул|јула|август|августа|септембар|септембра|октобар|октобра|новембар|новембра|децембар|децембра</afterbreak>-</rule>-<!--Не раздвајај у случајевима као на пр.: "у 1. степену сродства".-->-<rule break="no">-<beforebreak>\d+\.\s</beforebreak>-<afterbreak>степен(у)</afterbreak>-</rule>-<!-- German abbreviations -->-<rule break="no">-<beforebreak>\b(versch|d|Übers|usw|Ab[hkts]|ahd|Akk|aktual|allg|alltagsspr|altdt|alttest|amerikan|Anh|Ank|Anm|Art|Az|Bat|bayr|Bd|Bde|bearb|Bed|Bem|bes|bez|Bez|Bhf|bspw|btto|bw|bzw)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Српске скраћенице-->-<rule break="no">-<beforebreak>\b(одн|тј)\.\s+</beforebreak>-<afterbreak></afterbreak>-</rule>-<!--Раздвој после ових знакова, ако су праћени једним или већим бројем размака.-->-<rule break="yes">-<beforebreak>[\.!?…][\u0002|'|"|“|\)|\]|\}¹²³]?\s+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?…]['"“\p{Pe}\u00BB\u201D]?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Irish">-<rule break="no">-<beforebreak>\bPh\.</beforebreak>-<afterbreak>D\.</afterbreak>-</rule>-<!-- Don't split at e.g. "U.S.A." -->-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't split after a white-space followed by a single letter followed- by a dot followed by another whitespace. e.g. " p. " -->-<rule break="no">-<beforebreak>\s\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<!-- Don't split [.?!] when they're quoted -->-<rule break="no">-<beforebreak>['"][\.!?\u0085]['"]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Don't break after quote unless there's a capital letter- e.g.: "That's right!" he said. -->-<rule break="no">-<beforebreak>["']\s</beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\.\s</beforebreak>-<afterbreak>\d\d(\d\d)?</afterbreak>-</rule>-<rule break="no">-<beforebreak>(Ean|Fea|Már|Aib|Bea|Mei|Iúl|Lún|M\.?Fr|D\.?Fr|Sam|Nol)\.\s</beforebreak>-<afterbreak>\d\d(\d\d)?</afterbreak>-</rule>-<!-- English abbreviations - but these work globally for all languages -->-<rule break="no">-<beforebreak>\b(Mr|Mrs|Ms|No|pp|St|no|Sr|Jr|Bros|etc|vs|esp|[Ff]ig|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Okt|Nov|Dec|PhD|al|cf|Inc|Ms|Gen|Sen|Prof|Corp|Co|Ltd)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Latin abbreviations - but these work globally for all languages -->-<rule break="no">-<beforebreak>\b(sp|spp)\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>([Ll]ch|[Ll]gh|[Uu]imh)\.\s</beforebreak>-<afterbreak>\d</afterbreak>-</rule>-<rule break="no">-<beforebreak>\.i\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>Msc\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>Uas\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>Teo\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>m\.sh\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<!-- Break rules -->-<rule break="yes">-<beforebreak>[\.!?\u0085][\u0002|'|"|«|\)|\]|\}¹²³]?\s+</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?\u0085]['"\p{Pe}\u00BB\u201D]?</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?\u0085]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="GeneralImportant">-<!--Don't split up URLs.-->-<rule break="no">-<beforebreak>\bwww\.</beforebreak>-<afterbreak>\w</afterbreak>-</rule>-<!--Don't split up e-mail addresses.-->-<rule break="no">-<beforebreak>[\.!?]</beforebreak>-<afterbreak>\S*@</afterbreak>-</rule>-</languagerule>-<languagerule languagerulename="Arabic">-<rule break="no">-<beforebreak>\bwww\.</beforebreak>-<afterbreak>\w</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*…[\]\)]* </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>\p{Ps}[!?؟]+\p{Pe} </beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.!?؟…]+\p{Pe} </beforebreak>-<afterbreak>\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[«»"”']\s*</beforebreak>-<afterbreak>\s*\p{Ll}</afterbreak>-</rule>-<rule break="no">-<beforebreak>[«'"„][\.!?؟…]['"”»]\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.\s</beforebreak>-<afterbreak>\p{L}\.\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{L}\.</beforebreak>-<afterbreak>\p{L}\.</afterbreak>-</rule>-<rule break="yes">-<beforebreak>[^,،][\s]\p{L}{2}\.\s</beforebreak>-<afterbreak>\p{N}+\)\s</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\.\s]\p{L}{1,2}\.\s</beforebreak>-<afterbreak>[\p{N}\p{Ll}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>[\[\(]*\.\.\.[\]\)]* </beforebreak>-<afterbreak>[^\p{Lu}]</afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\s\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\b\p{Lu}\.\p{Lu}\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[^\.]\s[ابتقجحخدذصضعغفقكلمنهوىيءةأ١٢٣٤٥٦٧٨٩٠A-Z]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>[^\.]\s[\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652\u0653\u0654\u0655\u0656\u0640]\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="no">-<beforebreak>\(\p{Ll}+\.\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?؟…][«»\u00BB\u2019\u201D\u203A"'\p{Pe}\u0002¹²³]*\s</beforebreak>-<afterbreak></afterbreak>-</rule>-<rule break="yes">-<beforebreak>[\.!?؟…][«»'"\u00BB\u2019\u201D\u203A\p{Pe}\u0002]*</beforebreak>-<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>-</rule>-<rule break="yes">-<beforebreak>\s\p{L}[\.!?؟…]\s</beforebreak>-<afterbreak>\p{Lu}\p{Ll}</afterbreak>-</rule>-</languagerule>-</languagerules>-<maprules>-<languagemap languagepattern=".*" languagerulename="GeneralImportant"></languagemap>-<languagemap languagepattern="[a-z]{2,3}_one" languagerulename="ByLineBreak"></languagemap>-<languagemap languagepattern="[a-z]{2,3}_two" languagerulename="ByTwoLineBreaks"></languagemap>-<languagemap languagepattern="(EL|el).*" languagerulename="Greek"></languagemap>-<languagemap languagepattern="(PL|pl).*" languagerulename="Polish"></languagemap>-<languagemap languagepattern="(EN|en).*" languagerulename="English"></languagemap>-<languagemap languagepattern="(NL|nl).*" languagerulename="Dutch"></languagemap>-<languagemap languagepattern="(RO|ro).*" languagerulename="Romanian"></languagemap>-<languagemap languagepattern="(SK|sk).*" languagerulename="Slovak"></languagemap>-<languagemap languagepattern="(IS|is).*" languagerulename="Icelandic"></languagemap>-<languagemap languagepattern="(RU|ru).*" languagerulename="Russian"></languagemap>-<languagemap languagepattern="(SL|sl).*" languagerulename="Slovenian"></languagemap>-<languagemap languagepattern="(CA|ca).*" languagerulename="Catalan"></languagemap>-<languagemap languagepattern="(ES|es).*" languagerulename="Spanish"></languagemap>-<languagemap languagepattern="(DE|de).*" languagerulename="German"></languagemap>-<languagemap languagepattern="(DA|da).*" languagerulename="Danish"></languagemap>-<languagemap languagepattern="(EO|eo).*" languagerulename="Esperanto"></languagemap>-<languagemap languagepattern="(FR|fr).*" languagerulename="French"></languagemap>-<languagemap languagepattern="(UK|uk).*" languagerulename="Ukrainian"></languagemap>-<languagemap languagepattern="(BE|be).*" languagerulename="Belarusian"></languagemap>-<languagemap languagepattern="(GL|gl).*" languagerulename="Galician"></languagemap>-<languagemap languagepattern="(JA|ja).*" languagerulename="Ideographic"></languagemap>-<languagemap languagepattern="(ZH|zh).*" languagerulename="Ideographic"></languagemap>-<languagemap languagepattern="(BR|br).*" languagerulename="Breton"></languagemap>-<languagemap languagepattern="(PT|pt).*" languagerulename="Portuguese"></languagemap>-<languagemap languagepattern="(IT|it).*" languagerulename="Italian"></languagemap>-<languagemap languagepattern="(TA|ta).*" languagerulename="Tamil"></languagemap>-<languagemap languagepattern="(FA|fa).*" languagerulename="Persian"></languagemap>-<languagemap languagepattern="(GA|ga).*" languagerulename="Irish"></languagemap>-<languagemap languagepattern="(SR|sr).*" languagerulename="Serbian"></languagemap>-<languagemap languagepattern="(AR|ar).*" languagerulename="Arabic"></languagemap>-<languagemap languagepattern="(SV|sv).*" languagerulename="Generic"></languagemap>-<languagemap languagepattern="(LT|lt).*" languagerulename="Generic"></languagemap>-<languagemap languagepattern="(ML|ml).*" languagerulename="Generic"></languagemap>-<languagemap languagepattern="(TL|tl).*" languagerulename="Generic"></languagemap>-<languagemap languagepattern="(AST|ast).*" languagerulename="Generic"></languagemap>-<languagemap languagepattern=".*" languagerulename="Default"></languagemap>-</maprules>-</body>-</srx>diff --git a/core/src/spell/spell_checker.rs b/core/src/spell/spell_checker.rsdeleted file mode 100644index 57c4233b..00000000--- a/core/src/spell/spell_checker.rs+++ /dev/null@@ -1,261 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::spell::dictionary::EditStrategy;-use crate::spell::Dictionary;-use std::cmp::Ordering;-use std::collections::{BTreeMap, HashSet, VecDeque};--use super::dictionary::TermId;-use super::distance::LevenshteinDistance;--pub struct SpellChecker<T: EditStrategy> {- dict: Dictionary,- edit_strategy: T,- deletes: BTreeMap<String, Vec<TermId>>,-}--fn deletes_rec(- word: &str,- edit_distance: usize,- max_edit_distance: usize,- delete_words: &mut HashSet<String>,-) {- let edit_distance = edit_distance + 1;- if !word.is_empty() {- for (i, _) in word.char_indices() {- let delete = word- .chars()- .enumerate()- .filter(|(j, _)| *j != i)- .map(|(_, c)| c)- .collect::<String>();-- if !delete_words.contains(&delete) {- delete_words.insert(delete.clone());-- if edit_distance < max_edit_distance {- deletes_rec(&delete, edit_distance, max_edit_distance, delete_words);- }- }- }- }-}--fn deletes(term: &str, edit_strategy: &impl EditStrategy) -> Vec<String> {- let mut res = HashSet::new();- deletes_rec(term, 0, edit_strategy.dist().distance(), &mut res);-- res.into_iter().collect()-}--#[derive(Debug)]-struct Suggestion {- term: String,- distance: usize,- score: f64,-}--impl Ord for Suggestion {- fn cmp(&self, other: &Suggestion) -> Ordering {- let distance_cmp = self.distance.cmp(&other.distance);- if distance_cmp == Ordering::Equal {- return self- .score- .partial_cmp(&other.score)- .unwrap_or(Ordering::Equal)- .reverse();- }- distance_cmp- }-}--impl PartialOrd for Suggestion {- fn partial_cmp(&self, other: &Suggestion) -> Option<Ordering> {- Some(self.cmp(other))- }-}--impl PartialEq for Suggestion {- fn eq(&self, other: &Suggestion) -> bool {- self.distance == other.distance && self.score == other.score- }-}-impl Eq for Suggestion {}--impl<T: EditStrategy> SpellChecker<T> {- pub fn new(dict: Dictionary, edit_strategy: T) -> Self {- let mut deletes_map = BTreeMap::new();- for (term, id) in dict.terms() {- let term = term.to_ascii_lowercase();- for delete in deletes(&term, &edit_strategy) {- deletes_map.entry(delete).or_insert_with(Vec::new).push(*id);- }- }-- SpellChecker {- dict,- edit_strategy,- deletes: deletes_map,- }- }-- fn suggestions(&self, before: &[&str], term: &str, after: &[&str]) -> Vec<Suggestion> {- let mut suggestions = Vec::new();- let mut suggestion_terms = HashSet::new();-- let mut candidates = VecDeque::new();- candidates.push_back(term.to_ascii_lowercase());- for delete in deletes(term, &self.edit_strategy) {- candidates.push_back(delete.to_ascii_lowercase());- }- let max_dist = self.edit_strategy.distance_for_string(term);-- while let Some(candidate) = candidates.pop_front() {- if suggestion_terms.contains(&candidate) {- continue;- }-- let distance = LevenshteinDistance::compare(term, &candidate);- if distance <= max_dist {- if let Some(score) = self.dict.score(before, &candidate, after) {- suggestions.push(Suggestion {- term: candidate.clone(),- distance,- score,- });- suggestion_terms.insert(candidate.clone());- }-- if let Some(deletes) = self.deletes.get(&candidate) {- for delete in deletes {- candidates.push_back(self.dict.term(delete).unwrap().to_string());- }- }- }- }-- suggestions.sort();- suggestions- }-- pub fn correct(&self, before: &[&str], term: &str, after: &[&str]) -> Option<String> {- if self.dict.term_id(term).is_some() {- return None;- }-- if term.chars().any(|c| !c.is_ascii_alphabetic()) {- return None;- }-- self.suggestions(before, term, after)- .into_iter()- .map(|suggestion| suggestion.term)- .next()- }-}--#[cfg(test)]-mod tests {- use super::*;- use crate::spell::{- dictionary::{self, MaxEdit},- LogarithmicEdit,- };-- #[test]- fn simple_corrections() {- let dict = dictionary::build_from_str("this is a test test pest");-- let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));-- assert_eq!(- spell_checker.correct(&[], "tst", &[]),- Some("test".to_string())- );- assert_eq!(- spell_checker.correct(&[], "ths", &[]),- Some("this".to_string())- );- assert_eq!(- spell_checker.correct(&[], "thes", &[]),- Some("this".to_string())- );- assert_eq!(spell_checker.correct(&[], "is", &[]), None);- assert_eq!(- spell_checker.correct(&[], "dest", &[]),- Some("test".to_string())- );- }-- #[test]- fn correct_uncontained_word() {- let dict = dictionary::build_from_str("this is a test test");-- let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));-- assert_eq!(spell_checker.correct(&[], "what", &[]), None);- }-- #[test]- fn prioritise_low_distance_words() {- let dict = dictionary::build_from_str("this is a test test contest contest");-- let spell_checker = SpellChecker::new(dict, MaxEdit::new(4));-- assert_eq!(- spell_checker.correct(&[], "dest", &[]),- Some("test".to_string())- );- }-- #[test]- fn correct_sorting_multiple_hits() {- let dict = dictionary::build_from_str("the the the he");-- let spell_checker = SpellChecker::new(dict, LogarithmicEdit::new(4));-- assert_eq!(- spell_checker.correct(&[], "fhe", &[]),- Some("the".to_string())- );- }-- #[test]- fn dont_correct_non_alphabet() {- let dict = dictionary::build_from_str("this is a test c");-- let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));-- assert_eq!(spell_checker.correct(&[], "c++", &[]), None);- assert_eq!(spell_checker.correct(&[], "c#", &[]), None);- }-- #[test]- fn context_correction() {- let dict = dictionary::build_from_str("abraham lincoln was the 16th president of the united states. A wrong way to spell hist last name would be linculn linculn");-- let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));-- assert_eq!(- spell_checker.correct(&[], "lincln", &[]),- Some("linculn".to_string())- );- assert_eq!(- spell_checker.correct(&["abraham"], "lincln", &[]),- Some("lincoln".to_string())- );- }-}diff --git a/core/src/spell/splitter.rs b/core/src/spell/splitter.rsdeleted file mode 100644index 2d66409d..00000000--- a/core/src/spell/splitter.rs+++ /dev/null@@ -1,97 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::spell::Dictionary;--pub struct TermSplitter<'a> {- dict: &'a Dictionary,-}--impl<'a> TermSplitter<'a> {- pub fn new(dict: &'a Dictionary) -> Self {- TermSplitter { dict }- }- pub fn split(&self, text: &'a str) -> Vec<&'a str> {- let mut probs = vec![1.0];- let mut lasts: Vec<usize> = vec![0];-- for i in 1..text.len() + 1 {- let mut best_prob_k = 0.0;- let mut best_index = 0;-- for j in 0..i {- if !text.is_char_boundary(j) || !text.is_char_boundary(i) {- continue;- }-- if let Some(prob) = &self.dict.score(&[], &text[j..i], &[]) {- let new_prob = probs[j] * prob;- if new_prob > best_prob_k {- best_prob_k = new_prob;- best_index = j;- }- }- }-- probs.push(best_prob_k);- lasts.push(best_index);- }-- let mut words = Vec::new();- let mut i = text.len();-- while i > 0 {- if i == text.len() && lasts[i] == 0 {- break;- }- words.push(&text[lasts[i]..i]);- i = lasts[i];- }-- words.reverse();- words- }-}--#[cfg(test)]-mod tests {-- use crate::spell::dictionary;-- use super::*;-- #[test]- fn test_split() {- let dictionary = dictionary::build_from_str("wicked weather");-- assert_eq!(- TermSplitter::new(&dictionary).split("wickedweather"),- vec!["wicked", "weather"]- );-- assert_eq!(TermSplitter::new(&dictionary).split("wicked").len(), 0);- assert_eq!(TermSplitter::new(&dictionary).split("udlæg").len(), 0);- }-- #[test]- fn test_most_probable() {- let dictionary = dictionary::build_from_str("wicked weather wicked weather eat");-- assert_eq!(- TermSplitter::new(&dictionary).split("wickedweather"),- vec!["wicked", "weather"]- );- }-}diff --git a/core/src/spell/word2vec.rs b/core/src/spell/word2vec.rsdeleted file mode 100644index 20404738..00000000--- a/core/src/spell/word2vec.rs+++ /dev/null@@ -1,152 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--// Modified from: https://github.com/DimaKudosh/word2vec-use std::{- collections::HashMap,- fs::File,- io::{BufRead, BufReader},- path::Path,-};--use byteorder::{LittleEndian, ReadBytesExt};-use flate2::bufread::MultiGzDecoder;-use serde::{Deserialize, Serialize};--#[derive(thiserror::Error, Debug)]-pub enum Error {- #[error("Wrong header")]- WrongHeader,-- #[error("IO")]- IO(#[from] std::io::Error),-- #[error("Bincode")]- Bincode(#[from] bincode::Error),-}--struct WordVectorReader<R: BufRead> {- vocabulary_size: usize,- vector_size: usize,- reader: R,- ended_early: bool,- vectors_read: usize,-}--impl<R: BufRead> WordVectorReader<R> {- pub fn new_from_reader(mut reader: R) -> Result<WordVectorReader<R>, Error> {- // Read UTF8 header string from start of file- let mut header = String::new();- reader.read_line(&mut header)?;-- //Parse 2 integers, separated by whitespace- let header_info = header- .split_whitespace()- .filter_map(|x| x.parse::<usize>().ok())- .take(2)- .collect::<Vec<usize>>();- if header_info.len() != 2 {- return Err(Error::WrongHeader);- }-- //We've successfully read the header, ready to read vectors- Ok(WordVectorReader {- vocabulary_size: header_info[0],- vector_size: header_info[1],- vectors_read: 0,- ended_early: false,- reader,- })- }-}--impl<R: BufRead> Iterator for WordVectorReader<R> {- type Item = (String, Vec<f32>);-- fn next(&mut self) -> Option<(String, Vec<f32>)> {- if self.vectors_read == self.vocabulary_size {- return None;- }-- // Read the bytes of the word string- let mut word_bytes: Vec<u8> = Vec::new();- if self.reader.read_until(b' ', &mut word_bytes).is_err() {- // End the stream if a read error occured- self.ended_early = true;- return None;- }-- // trim newlines, some vector files have newlines in front of a new word, others don't- let word = match String::from_utf8(word_bytes) {- Err(_) => {- self.ended_early = true;- return None;- }- Ok(word) => word.trim().into(),- };-- // Read floats of the vector- let mut vector: Vec<f32> = Vec::with_capacity(self.vector_size);- for _ in 0..self.vector_size {- match self.reader.read_f32::<LittleEndian>() {- Err(_) => {- self.ended_early = true;- return None;- }- Ok(value) => vector.push(value),- }- }-- self.vectors_read += 1;- Some((word, vector))- }-}--#[derive(Serialize, Deserialize, Debug, Clone)]-pub struct WordVec(Vec<f32>);-impl WordVec {- fn magnitude(&self) -> f32 {- self.0.iter().map(|f| f.powf(2.0)).sum::<f32>().sqrt()- }-- pub(crate) fn sim(&self, other: &WordVec) -> f32 {- self.0- .iter()- .zip(other.0.iter())- .map(|(a, b)| a * b)- .sum::<f32>()- / (self.magnitude() * other.magnitude())- }-}--#[derive(Serialize, Deserialize)]-pub struct Word2Vec {- vectors: HashMap<String, WordVec>,-}--impl Word2Vec {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {- let reader = BufReader::new(MultiGzDecoder::new(BufReader::new(File::open(path)?)));- let reader = WordVectorReader::new_from_reader(reader)?;- let vectors: HashMap<_, _> = reader.map(|(word, vec)| (word, WordVec(vec))).collect();-- Ok(Self { vectors })- }-- pub(crate) fn get(&self, word: &str) -> Option<&WordVec> {- self.vectors.get(word)- }-}diff --git a/core/src/summarizer.rs b/core/src/summarizer.rsindex f571a8f0..16bd3f89 100644--- a/core/src/summarizer.rs+++ b/core/src/summarizer.rs@@ -29,7 +29,6 @@ use tokenizers::{PaddingParams, TruncationParams};use crate::{ceil_char_boundary,llm_utils::{self, ClonableTensor},- spell::word2vec::{Word2Vec, WordVec},};#[derive(thiserror::Error, Debug)]@@ -43,9 +42,6 @@ pub enum Error {#[error("IO")]Io(#[from] std::io::Error),- #[error("Word2vec")]- Word2Vec(#[from] crate::spell::word2vec::Error),-#[error("Unexpected output type")]UnexpectedOutputType,}@@ -178,51 +174,6 @@ trait PassageScorer {fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32;}-impl PassageScorer for Word2Vec {- type QueryEmbedding = Vec<WordVec>;-- type PassageEmbedding = Self::QueryEmbedding;-- fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {- let res: Self::QueryEmbedding = query- .split_whitespace()- .filter_map(|word| self.get(word).cloned())- .collect();-- if res.is_empty() {- return None;- }-- Some(res)- }-- fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {- let res: Self::PassageEmbedding = passage- .split_whitespace()- .filter_map(|word| self.get(word).cloned())- .collect();-- if res.is_empty() {- return None;- }-- Some(res)- }-- fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {- let mut score = 0.0;- let mut count = 0;-- for passage_vec in passage {- score += query.iter().map(|vec| vec.sim(passage_vec)).sum::<f32>();-- count += 1;- }-- score / count as f32- }-}-pub struct ExtractiveSummarizer {passage_scorer: DualEncoder,top_n_passages: usize,diff --git a/core/src/web_spell/error_model.rs b/core/src/web_spell/error_model.rsnew file mode 100644index 00000000..a44e68c3--- /dev/null+++ b/core/src/web_spell/error_model.rs@@ -0,0 +1,197 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use serde::{Deserialize, Serialize};++use super::Result;+use std::{+ collections::HashMap,+ fs::OpenOptions,+ io::{BufReader, BufWriter},+ path::Path,+};++#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+pub enum ErrorType {+ Insertion(char),+ Deletion(char),+ Substitution(char, char),+ Transposition(char, char),+}++#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+pub struct ErrorSequence(Vec<ErrorType>);++fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {+ if a == b {+ return None;+ }++ let a_len = a.chars().count();+ let b_len = b.chars().count();+ let mut dp = vec![vec![0; b_len + 1]; a_len + 1];++ for i in 0..=a_len {+ for j in 0..=b_len {+ if i == 0 {+ dp[i][j] = j;+ } else if j == 0 {+ dp[i][j] = i;+ } else {+ let cost = if a.chars().nth(i - 1) == b.chars().nth(j - 1) {+ 0+ } else {+ 1+ };+ dp[i][j] = std::cmp::min(+ std::cmp::min(dp[i - 1][j] + 1, dp[i][j - 1] + 1),+ dp[i - 1][j - 1] + cost,+ );+ }+ }+ }++ let mut i = a_len;+ let mut j = b_len;+ let mut errors = Vec::new();++ while i > 0 && j > 0 {+ let cost = if a.chars().nth(i - 1) == b.chars().nth(j - 1) {+ 0+ } else {+ 1+ };+ if dp[i][j] == dp[i - 1][j - 1] + cost {+ if cost == 1 {+ errors.push(ErrorType::Substitution(+ a.chars().nth(i - 1).unwrap(),+ b.chars().nth(j - 1).unwrap(),+ ));+ }+ i -= 1;+ j -= 1;+ } else if dp[i][j] == dp[i - 1][j] + 1 {+ errors.push(ErrorType::Deletion(a.chars().nth(i - 1).unwrap()));+ i -= 1;+ } else {+ errors.push(ErrorType::Insertion(b.chars().nth(j - 1).unwrap()));+ j -= 1;+ }+ }++ while i > 0 {+ errors.push(ErrorType::Deletion(a.chars().nth(i - 1).unwrap()));+ i -= 1;+ }++ while j > 0 {+ errors.push(ErrorType::Insertion(b.chars().nth(j - 1).unwrap()));+ j -= 1;+ }++ if !errors.is_empty() {+ Some(ErrorSequence(errors))+ } else {+ None+ }+}++#[derive(Debug, Serialize, Deserialize)]+pub struct ErrorModel {+ errors: HashMap<ErrorSequence, u64>,+ total: u64,+}++impl Default for ErrorModel {+ fn default() -> Self {+ Self::new()+ }+}++impl ErrorModel {+ pub fn new() -> Self {+ Self {+ errors: HashMap::new(),+ total: 0,+ }+ }++ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {+ let file = OpenOptions::new()+ .write(true)+ .create(true)+ .truncate(true)+ .open(path)?;++ let wrt = BufWriter::new(file);++ serde_json::to_writer_pretty(wrt, &self)?;++ Ok(())+ }++ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let file = OpenOptions::new().read(true).open(path)?;++ let rdr = BufReader::new(file);++ Ok(serde_json::from_reader(rdr)?)+ }++ pub fn add(&mut self, a: &str, b: &str) {+ if let Some(errors) = possible_errors(a, b) {+ *self.errors.entry(errors).or_insert(0) += 1;+ }+ }++ pub fn prob(&self, error: &ErrorSequence) -> f64 {+ let count = self.errors.get(error).unwrap_or(&0);+ *count as f64 / self.total as f64+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_possible_errors() {+ assert_eq!(possible_errors("hello", "hello"), None);++ assert_eq!(+ possible_errors("hello", "helo"),+ Some(ErrorSequence(vec![ErrorType::Deletion('l')]))+ );++ assert_eq!(+ possible_errors("hello", "hellol"),+ Some(ErrorSequence(vec![ErrorType::Insertion('l')]))+ );++ assert_eq!(+ possible_errors("hello", "heo"),+ Some(ErrorSequence(vec![+ ErrorType::Deletion('l'),+ ErrorType::Deletion('l')+ ]))+ );++ assert_eq!(+ possible_errors("hello", "helli"),+ Some(ErrorSequence(vec![ErrorType::Substitution('o', 'i')]))+ );+ }+}diff --git a/core/src/web_spell/mod.rs b/core/src/web_spell/mod.rsnew file mode 100644index 00000000..a50356c5--- /dev/null+++ b/core/src/web_spell/mod.rs@@ -0,0 +1,148 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! This module contains the spell checker. It is based on the paper+//! http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf+//! from google.+mod error_model;+pub mod spell_checker;+mod stupid_backoff;+mod term_freqs;+mod trainer;++pub use error_model::ErrorModel;+pub use stupid_backoff::StupidBackoff;+pub use term_freqs::TermDict;+pub use trainer::FirstTrainer;+pub use trainer::FirstTrainerResult;+pub use trainer::SecondTrainer;++use fst::Streamer;+use std::ops::Range;++use crate::floor_char_boundary;+use itertools::intersperse;+use serde::{Deserialize, Serialize};++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("IO error: {0}")]+ Io(#[from] std::io::Error),++ #[error("FST error: {0}")]+ Fst(#[from] fst::Error),++ #[error("Serde error: {0}")]+ Serde(#[from] serde_json::Error),++ #[error("Bincode error: {0}")]+ Bincode(#[from] bincode::Error),+}++pub type Result<T, E = Error> = std::result::Result<T, E>;++#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]+pub struct Correction {+ original: String,+ pub terms: Vec<CorrectionTerm>,+}++#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]+pub enum CorrectionTerm {+ Corrected(String),+ NotCorrected(String),+}++impl From<Correction> for String {+ fn from(correction: Correction) -> Self {+ intersperse(+ correction.terms.into_iter().map(|term| match term {+ CorrectionTerm::Corrected(correction) => correction,+ CorrectionTerm::NotCorrected(orig) => orig,+ }),+ " ".to_string(),+ )+ .collect()+ }+}++impl Correction {+ pub fn empty(original: String) -> Self {+ Self {+ original,+ terms: Vec::new(),+ }+ }++ pub fn push(&mut self, term: CorrectionTerm) {+ self.terms.push(term);+ }++ pub fn is_all_orig(&self) -> bool {+ self.terms+ .iter()+ .all(|term| matches!(term, CorrectionTerm::NotCorrected(_)))+ }+}++pub fn sentence_ranges(text: &str) -> Vec<Range<usize>> {+ let mut res = Vec::new();+ let mut last_start = 0;++ // We should really do something more clever than this.+ // Tried using `SRX`[https://docs.rs/srx/latest/srx/] but it was a bit too slow.+ for (end, _) in text+ .char_indices()+ .filter(|(_, c)| matches!(c, '.' | '\n' | '?' | '!'))+ {+ res.push(last_start..end + 1);+ last_start = floor_char_boundary(text, end + 2);+ }++ res+}++pub fn tokenize(text: &str) -> Vec<String> {+ text.to_lowercase()+ .split_whitespace()+ .filter(|s| {+ !s.chars()+ .any(|c| !c.is_ascii_alphanumeric() && c != '-' && c != '_')+ })+ .map(|s| s.to_string())+ .collect()+}+pub struct MergePointer<'a> {+ pub term: String,+ pub freq: u64,+ pub stream: fst::map::Stream<'a>,+ pub is_finished: bool,+}++impl<'a> MergePointer<'a> {+ pub fn advance(&mut self) -> bool {+ self.is_finished = self+ .stream+ .next()+ .map(|(term, freq)| {+ self.term = std::str::from_utf8(term).unwrap().to_string();+ self.freq = freq;+ })+ .is_none();++ !self.is_finished+ }+}diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rsnew file mode 100644index 00000000..bfa7dc6d--- /dev/null+++ b/core/src/web_spell/spell_checker.rs@@ -0,0 +1,15 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rsnew file mode 100644index 00000000..518301cf--- /dev/null+++ b/core/src/web_spell/stupid_backoff.rs@@ -0,0 +1,288 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::{tokenize, MergePointer, Result};+use std::{+ collections::BTreeMap,+ fs::{File, OpenOptions},+ io::BufWriter,+ path::Path,+};++use fst::IntoStreamer;+use serde::{Deserialize, Serialize};++const DISCOUNT: f64 = 0.4;++#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]+pub struct Ngram {+ terms: Vec<String>,+}++pub struct StoredNgram {+ combined: String,+}++impl From<Ngram> for StoredNgram {+ fn from(ngram: Ngram) -> Self {+ Self {+ combined: ngram.terms.join(" "),+ }+ }+}++impl AsRef<[u8]> for StoredNgram {+ fn as_ref(&self) -> &[u8] {+ self.combined.as_bytes()+ }+}++pub struct StupidBackoffTrainer {+ max_ngram_size: usize,+ ngrams: BTreeMap<Ngram, u64>,+ n_counts: Vec<u64>,+}++impl StupidBackoffTrainer {+ pub fn new(max_ngram_size: usize) -> Self {+ Self {+ max_ngram_size,+ ngrams: BTreeMap::new(),+ n_counts: vec![0; max_ngram_size],+ }+ }++ pub fn train(&mut self, tokens: &[String]) {+ for window in tokens.windows(self.max_ngram_size) {+ for i in 1..window.len() {+ let ngram = Ngram {+ terms: window[..i].to_vec(),+ };++ self.ngrams+ .entry(ngram)+ .and_modify(|e| *e += 1)+ .or_insert(1);++ self.n_counts[i - 1] += 1;+ }+ }+ }++ pub fn build<P: AsRef<Path>>(self, path: P) -> Result<()> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref().join("ngrams.bin"))?;++ let wtr = BufWriter::new(file);++ let mut builder = fst::MapBuilder::new(wtr)?;++ for (ngram, freq) in self.ngrams {+ builder.insert(StoredNgram::from(ngram), freq)?;+ }++ builder.finish()?;++ bincode::serialize_into(+ File::create(path.as_ref().join("n_counts.bin"))?,+ &self.n_counts,+ )?;++ Ok(())+ }+}++pub struct StupidBackoff {+ ngrams: fst::Map<memmap::Mmap>,+ n_counts: Vec<u64>,+}++impl StupidBackoff {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let ngrams = fst::Map::new(mmap)?;++ let n_counts = bincode::deserialize_from(File::open(path.as_ref().join("n_counts.bin"))?)?;++ Ok(Self { ngrams, n_counts })+ }++ pub fn merge<P: AsRef<Path>>(models: Vec<Self>, path: P) -> Result<Self> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }++ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref().join("ngrams.bin"))?;++ let n_counts = models+ .iter()+ .fold(vec![0; models[0].n_counts.len()], |mut acc, m| {+ for (i, n) in m.n_counts.iter().enumerate() {+ acc[i] += n;+ }++ acc+ });++ let wtr = BufWriter::new(file);+ let mut builder = fst::MapBuilder::new(wtr)?;++ let mut pointers: Vec<_> = models+ .iter()+ .map(|d| MergePointer {+ term: String::new(),+ freq: 0,+ stream: d.ngrams.stream(),+ is_finished: false,+ })+ .collect();++ for pointer in pointers.iter_mut() {+ pointer.advance();+ }++ while pointers.iter().any(|p| !p.is_finished) {+ let mut min_pointer: Option<&MergePointer<'_>> = None;++ for pointer in pointers.iter() {+ if pointer.is_finished {+ continue;+ }++ if let Some(min) = min_pointer {+ if pointer.term < min.term {+ min_pointer = Some(pointer);+ }+ } else {+ min_pointer = Some(pointer);+ }+ }++ if let Some(min_pointer) = min_pointer {+ let term = min_pointer.term.clone();+ let mut freq = 0;++ for pointer in pointers.iter_mut() {+ if pointer.is_finished {+ continue;+ }++ if pointer.term == term {+ freq += pointer.freq;+ pointer.advance();+ }+ }++ builder.insert(term, freq)?;+ }+ }++ builder.finish()?;++ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };++ Ok(Self {+ ngrams: fst::Map::new(mmap)?,+ n_counts,+ })+ }++ pub fn freq(&self, words: &[String]) -> Option<u64> {+ if words.len() >= self.ngrams.len() || words.is_empty() {+ return None;+ }++ let ngram = StoredNgram {+ combined: words.join(" "),+ };++ self.ngrams.get(ngram)+ }++ pub fn log_prob<S: NextWordsStrategy>(&self, words: &[String], strat: S) -> f64 {+ if words.len() >= self.ngrams.len() || words.is_empty() {+ return 0.0;+ }++ if let Some(freq) = self.freq(words) {+ (freq as f64 / self.n_counts[words.len() - 1] as f64).log2()+ } else {+ let mut strat = strat;+ DISCOUNT.log2() + self.log_prob(strat.next_words(words), strat)+ }+ }++ pub fn contexts(&self, word: &str) -> Vec<Vec<String>> {+ let q = " ".to_string() + word + " ";+ let automaton = fst::automaton::Subsequence::new(q.as_str());++ self.ngrams+ .search(automaton)+ .into_stream()+ .into_str_keys()+ .unwrap_or_default()+ .into_iter()+ .map(|s| tokenize(&s))+ .collect()+ }+}++pub trait NextWordsStrategy {+ fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String];+}++pub struct LeftToRight;++impl NextWordsStrategy for LeftToRight {+ fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {+ &words[1..]+ }+}++pub struct RightToLeft;++impl NextWordsStrategy for RightToLeft {+ fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {+ &words[..words.len() - 1]+ }+}++#[derive(Default)]+pub struct IntoMiddle {+ last_right: bool,+}++impl NextWordsStrategy for IntoMiddle {+ fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {+ let res = if self.last_right {+ &words[..words.len() - 1]+ } else {+ &words[1..]+ };++ self.last_right = !self.last_right;++ res+ }+}diff --git a/core/src/web_spell/term_freqs.rs b/core/src/web_spell/term_freqs.rsnew file mode 100644index 00000000..0e420bb4--- /dev/null+++ b/core/src/web_spell/term_freqs.rs@@ -0,0 +1,477 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::{MergePointer, Result};+use fst::{IntoStreamer, Streamer};+use serde::{Deserialize, Serialize};+use std::{+ collections::{BTreeMap, BinaryHeap},+ fs::{File, OpenOptions},+ io::BufWriter,+ path::{Path, PathBuf},+};+use uuid::Uuid;++struct DictBuilder {+ map: BTreeMap<String, u64>,+}++impl Default for DictBuilder {+ fn default() -> Self {+ Self::new()+ }+}++impl DictBuilder {+ fn new() -> Self {+ Self {+ map: BTreeMap::new(),+ }+ }++ fn insert(&mut self, term: &str) {+ self.map+ .entry(term.to_string())+ .and_modify(|e| *e += 1)+ .or_insert(1);+ }++ fn build<P: AsRef<Path>>(self, path: P) -> Result<StoredDict> {+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref())?;++ let wtr = BufWriter::new(file);++ let mut builder = fst::MapBuilder::new(wtr)?;++ for (term, freq) in self.map {+ builder.insert(term, freq)?;+ }++ builder.finish()?;++ StoredDict::open(path)+ }+}++struct StoredDict {+ map: fst::Map<memmap::Mmap>,+ path: PathBuf,+}++impl StoredDict {+ fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };++ Ok(Self {+ map: fst::Map::new(mmap)?,+ path: path.as_ref().to_path_buf(),+ })+ }++ fn merge<P: AsRef<Path>>(dicts: Vec<Self>, path: P) -> Result<Self> {+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref())?;++ let wtr = BufWriter::new(file);+ let mut builder = fst::MapBuilder::new(wtr)?;++ let mut pointers: Vec<_> = dicts+ .iter()+ .map(|d| MergePointer {+ term: String::new(),+ freq: 0,+ stream: d.map.stream(),+ is_finished: false,+ })+ .collect();++ for pointer in pointers.iter_mut() {+ pointer.advance();+ }++ while pointers.iter().any(|p| !p.is_finished) {+ let mut min_pointer: Option<&MergePointer<'_>> = None;++ for pointer in pointers.iter() {+ if pointer.is_finished {+ continue;+ }++ if let Some(min) = min_pointer {+ if pointer.term < min.term {+ min_pointer = Some(pointer);+ }+ } else {+ min_pointer = Some(pointer);+ }+ }++ if let Some(min_pointer) = min_pointer {+ let term = min_pointer.term.clone();+ let mut freq = 0;++ for pointer in pointers.iter_mut() {+ if pointer.is_finished {+ continue;+ }++ if pointer.term == term {+ freq += pointer.freq;+ pointer.advance();+ }+ }++ builder.insert(term, freq)?;+ }+ }++ builder.finish()?;++ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };++ Ok(StoredDict {+ map: fst::Map::new(mmap)?,+ path: path.as_ref().to_path_buf(),+ })+ }+}++#[derive(Default, Serialize, Deserialize)]+struct Metadata {+ dicts: Vec<Uuid>,+}++pub struct TermDict {+ builder: DictBuilder,+ stored: Vec<StoredDict>,+ path: PathBuf,+ metadata: Metadata,+}++impl TermDict {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ if path.as_ref().exists() {+ let file = File::open(path.as_ref().join("meta.json"))?;+ let metadata: Metadata = serde_json::from_reader(file)?;++ let mut stored = Vec::new();++ for uuid in metadata.dicts.iter() {+ stored.push(StoredDict::open(+ path.as_ref().join(format!("{}.dict", uuid)),+ )?);+ }++ Ok(Self {+ builder: DictBuilder::new(),+ stored,+ path: path.as_ref().to_path_buf(),+ metadata,+ })+ } else {+ std::fs::create_dir_all(path.as_ref())?;++ let s = Self {+ builder: DictBuilder::new(),+ stored: Vec::new(),+ path: path.as_ref().to_path_buf(),+ metadata: Metadata::default(),+ };+ s.save_meta()?;++ Ok(s)+ }+ }++ pub fn insert(&mut self, term: &str) {+ if term.len() <= 1 {+ return;+ }++ if term.len() > 100 {+ return;+ }++ if term.contains(' ') {+ return;+ }++ let punctuation_percentage =+ term.chars().filter(|c| c.is_ascii_punctuation()).count() as f64 / term.len() as f64;++ if punctuation_percentage > 0.5 {+ return;+ }++ let non_alphabetic_percentage =+ term.chars().filter(|c| !c.is_alphabetic()).count() as f64 / term.len() as f64;++ if non_alphabetic_percentage > 0.25 {+ return;+ }++ self.builder.insert(term);+ }++ pub fn commit(&mut self) -> Result<()> {+ let builder = std::mem::take(&mut self.builder);++ let uuid = uuid::Uuid::new_v4();++ let stored = builder.build(self.path.join(format!("{}.dict", uuid)))?;++ self.metadata.dicts.push(uuid);+ self.save_meta()?;++ self.stored.push(stored);++ Ok(())+ }++ fn save_meta(&self) -> Result<()> {+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(self.path.join("meta.json"))?;++ serde_json::to_writer_pretty(file, &self.metadata)?;++ Ok(())+ }++ pub fn merge_dicts(&mut self) -> Result<()> {+ if self.stored.len() <= 1 {+ return Ok(());+ }++ let uuid = uuid::Uuid::new_v4();++ let merged = StoredDict::merge(+ std::mem::take(&mut self.stored),+ self.path.join(format!("{}.dict", uuid)),+ )?;+ self.metadata.dicts.clear();++ self.metadata.dicts.push(uuid);+ self.save_meta()?;++ self.stored.push(merged);++ Ok(())+ }++ pub fn freq(&self, term: &str) -> Option<u64> {+ let mut freqs = None;++ for stored in self.stored.iter() {+ if let Some(freq) = stored.map.get(term) {+ match freqs {+ None => freqs = Some(freq),+ Some(f) => freqs = Some(f + freq),+ }+ }+ }++ freqs+ }++ pub fn prune(&mut self, top_n_terms: usize) -> Result<()> {+ let mut top_term_freqs: BinaryHeap<u64> = BinaryHeap::new();++ for stored in self.stored.iter() {+ let mut stream = stored.map.stream();++ while let Some((_, freq)) = stream.next() {+ if top_term_freqs.len() < top_n_terms {+ top_term_freqs.push(freq);+ } else if let Some(mut min) = top_term_freqs.peek_mut() {+ if freq > *min {+ *min = freq;+ }+ }+ }+ }++ if top_term_freqs.len() < top_n_terms {+ return Ok(());+ }++ let lowest = top_term_freqs.into_sorted_vec().pop().unwrap();++ self.metadata.dicts.clear();+ for stored in self.stored.iter_mut() {+ let uuid = uuid::Uuid::new_v4();+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(self.path.join(format!("{}.dict", uuid)))?;++ let wtr = BufWriter::new(file);++ let mut builder = fst::MapBuilder::new(wtr)?;++ let mut stream = stored.map.stream();+ while let Some((term, freq)) = stream.next() {+ if freq >= lowest {+ builder.insert(term, freq)?;+ }+ }++ self.metadata.dicts.push(uuid);++ builder.finish()?;++ *stored = StoredDict::open(self.path.join(format!("{}.dict", uuid)))?;+ }++ self.save_meta()?;++ Ok(())+ }++ pub fn terms(&self) -> Vec<String> {+ let mut terms = Vec::new();++ for stored in self.stored.iter() {+ let mut stream = stored.map.stream();++ while let Some((term, _)) = stream.next() {+ terms.push(std::str::from_utf8(term).unwrap().to_string());+ }+ }++ terms+ }++ pub fn search(&self, term: &str, max_edit_distance: u32) -> Vec<String> {+ let mut res = Vec::new();++ for stored in self.stored.iter() {+ if let Ok(automaton) = fst::automaton::Levenshtein::new(term, max_edit_distance) {+ if let Ok(mut s) = stored.map.search(automaton).into_stream().into_str_keys() {+ res.append(&mut s);+ }+ }+ }++ res+ }++ pub fn merge(&mut self, other: Self) -> Result<()> {+ for stored in other.stored {+ let uuid = uuid::Uuid::new_v4();+ let new_path = self.path.join(format!("{}.dict", uuid));+ std::fs::rename(stored.path, &new_path)?;++ self.metadata.dicts.push(uuid);+ self.save_meta()?;++ self.stored.push(StoredDict::open(new_path)?);+ }++ Ok(())+ }++ pub(crate) fn path(&self) -> &Path {+ &self.path+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::gen_temp_path;+ use anyhow::Result;++ #[test]+ fn test_term_dict() -> Result<()> {+ let mut dict = TermDict::open(gen_temp_path())?;++ dict.insert("foo");+ dict.insert("bar");+ dict.insert("baz");+ dict.insert("foo");+ dict.insert("bar");+ dict.insert("foo");++ dict.commit()?;++ dict.insert("foo");+ dict.insert("bar");+ dict.insert("baz");+ dict.insert("foo");+ dict.insert("bar");+ dict.insert("foo");++ dict.commit()?;++ dict.merge_dicts()?;++ assert_eq!(dict.stored.len(), 1);++ assert_eq!(dict.freq("foo"), Some(6));+ assert_eq!(dict.freq("bar"), Some(4));+ assert_eq!(dict.freq("baz"), Some(2));++ Ok(())+ }++ #[test]+ fn reopen() {+ let path = gen_temp_path();++ {+ let mut dict = TermDict::open(&path).unwrap();++ dict.insert("foo");+ dict.insert("bar");+ dict.insert("baz");+ dict.insert("foo");+ dict.insert("bar");+ dict.insert("foo");++ dict.commit().unwrap();+ }++ {+ let mut dict = TermDict::open(&path).unwrap();++ dict.insert("foo");+ dict.insert("bar");+ dict.insert("baz");+ dict.insert("foo");+ dict.insert("bar");+ dict.insert("foo");++ dict.commit().unwrap();+ }++ {+ let dict = TermDict::open(&path).unwrap();++ assert_eq!(dict.stored.len(), 2);++ assert_eq!(dict.freq("foo"), Some(6));+ assert_eq!(dict.freq("bar"), Some(4));+ assert_eq!(dict.freq("baz"), Some(2));+ }+ }+}diff --git a/core/src/web_spell/trainer.rs b/core/src/web_spell/trainer.rsnew file mode 100644index 00000000..46a27118--- /dev/null+++ b/core/src/web_spell/trainer.rs@@ -0,0 +1,200 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use rayon::prelude::*;++use crate::web_spell::{error_model::ErrorModel, stupid_backoff::IntoMiddle};++use super::{stupid_backoff::StupidBackoffTrainer, tokenize, Result};+use std::{+ collections::HashMap,+ path::{Path, PathBuf},+};++use super::{stupid_backoff::StupidBackoff, term_freqs::TermDict};++pub struct FirstTrainer {+ term_dict: TermDict,+ lm_model: StupidBackoffTrainer,++ path: PathBuf,+}++impl FirstTrainer {+ pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {+ Ok(Self {+ term_dict: TermDict::open(path.as_ref().join("term_dict"))?,+ lm_model: StupidBackoffTrainer::new(3),+ path: path.as_ref().to_path_buf(),+ })+ }++ pub fn add(&mut self, text: &str) {+ let tokens = tokenize(text);+ for term in &tokens {+ self.term_dict.insert(term);+ }++ self.lm_model.train(&tokens);+ }++ pub fn next_training_step(mut self) -> Result<FirstTrainerResult> {+ self.term_dict.commit()?;+ self.term_dict.merge_dicts()?;+ self.lm_model.build(self.path.join("stupid_backoff"))?;++ Ok(FirstTrainerResult {+ term_dict: self.term_dict,+ lm_model: StupidBackoff::open(self.path.join("stupid_backoff"))?,+ })+ }+}++pub struct FirstTrainerResult {+ term_dict: TermDict,+ lm_model: StupidBackoff,+}++/// Constructs the error model from the first training step.+pub struct SecondTrainer {+ term_dict: TermDict,+ lm_model: StupidBackoff,++ path: PathBuf,+}++impl SecondTrainer {+ /// Panics if vec is empty.+ pub fn new<P: AsRef<Path>>(first_steps: Vec<FirstTrainerResult>, path: P) -> Result<Self> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }++ let (lm_models, mut term_dicts): (Vec<_>, Vec<_>) = first_steps+ .into_iter()+ .map(|r| (r.lm_model, r.term_dict))+ .unzip();++ let lm_model = StupidBackoff::merge(lm_models, path.as_ref().join("stupid_backoff"))?;++ let mut term_dict = term_dicts.pop().unwrap();++ for other in term_dicts {+ term_dict.merge(other)?;+ }+ tracing::debug!("merged term dicts");++ std::fs::rename(term_dict.path(), path.as_ref().join("term_dict"))?;+ drop(term_dict);++ let mut term_dict = TermDict::open(path.as_ref().join("term_dict"))?;+ term_dict.merge_dicts()?;+ term_dict.prune(10_000_000)?;++ Ok(Self {+ term_dict,+ lm_model,+ path: path.as_ref().to_path_buf(),+ })+ }++ pub fn train(self) -> Result<()> {+ let terms = self.term_dict.terms();++ let errors: Vec<_> = terms+ .into_par_iter()+ .map(|term| {+ // one edit for words of+ // up to four characters, two edits for up to twelve+ // characters, and three for longer+ let max_edit_distance = if term.len() <= 4 {+ 1+ } else if term.len() <= 12 {+ 2+ } else {+ 3+ };++ let possible_errors = self+ .term_dict+ .search(&term, max_edit_distance)+ .into_iter()+ .filter(|incorrect| {+ incorrect != &term+ && 10 * self.term_dict.freq(incorrect).unwrap_or_default()+ > self.term_dict.freq(&term).unwrap_or_default()+ })+ .collect::<Vec<_>>();++ (term, possible_errors)+ })+ .filter(|(_, errors)| !errors.is_empty())+ .map(|(term, possible_errors)| {+ let contexts = self.lm_model.contexts(&term);++ let best_terms = contexts+ .into_iter()+ .filter(|c| c.len() != 3)+ .map(|context| {+ let most_probable = possible_errors+ .iter()+ .chain(vec![&term])+ .map(|t| {+ let mut words = context.clone();+ words[1] = t.clone();++ (self.lm_model.log_prob(&words, IntoMiddle::default()), t)+ })+ .max_by(|(a, _), (b, _)| a.total_cmp(b))+ .unwrap()+ .1+ .clone();++ most_probable+ })+ .collect::<Vec<_>>();++ // count number of contexts for each best_term+ let mut counts: HashMap<String, u64> = HashMap::new();++ for t in best_terms {+ *counts.entry(t).or_insert(0) += 1;+ }++ (term, counts)+ })+ .collect();++ let mut error_model = ErrorModel::new();++ for (term, correction) in errors+ .into_iter()+ .flat_map(|(term, correction)| {+ correction+ .keys()+ .map(|other| (term.clone(), other.clone()))+ .collect::<Vec<_>>()+ })+ .filter(|(a, b)| a != b)+ {+ error_model.add(&term, &correction);+ }++ error_model.save(self.path.join("error_model.json"))?;++ Ok(())+ }+}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex b23c244d..be1879ae 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -475,6 +475,10 @@ impl Html {Ok(res)}+ pub fn lang(&self) -> Option<&'_ Lang> {+ self.lang.as_ref()+ }+pub fn canonical_url(&self) -> Option<Url> {let mut canonical_url = None;diff --git a/frontend/src/lib/components/Button.svelte b/frontend/src/lib/components/Button.svelteindex 85e166f6..f382d1e2 100644--- a/frontend/src/lib/components/Button.svelte+++ b/frontend/src/lib/components/Button.svelte@@ -18,6 +18,7 @@export let pale: boolean = false;export let padding: boolean = true;export let kind: Kind = 'primary';+ export let _class: string = '';</script><button@@ -82,6 +83,7 @@kind == 'warning' && 'bg-warning text-warning-content hover:bg-warning-focus',kind == 'error' && 'bg-error text-error-content hover:bg-error-focus',],+ _class,)}><slot />diff --git a/frontend/src/lib/components/Searchbar.svelte b/frontend/src/lib/components/Searchbar.svelteindex 1d048ea2..295d54f9 100644--- a/frontend/src/lib/components/Searchbar.svelte+++ b/frontend/src/lib/components/Searchbar.svelte@@ -93,7 +93,7 @@<labelfor="searchbar"class={twJoin(- 'group relative grid w-full grid-cols-[auto_1fr_auto] items-center rounded-3xl border border-base-400 py-0.5 pl-5 pr-0.5 transition focus-within:shadow',+ 'group relative grid w-full grid-cols-[auto_1fr_auto] items-center rounded-3xl border border-base-400 pl-5 transition focus-within:shadow',hasFocus && suggestions.length > 0 && 'rounded-b-none',hasFocus && 'shadow',)}@@ -121,7 +121,9 @@bind:value={query}on:keydown={onKeydown}/>- <Button type="submit">search</Button>+ <div class="h-full py-0.5 pr-0.5">+ <Button _class="py-0 h-full" type="submit">search</Button>+ </div>{#if suggestions.length > 0}<div class="absolute inset-x-5 bottom-px hidden h-px bg-base-300 group-focus-within:block" />diff --git a/frontend/src/routes/+page.svelte b/frontend/src/routes/+page.svelteindex 78fa71b0..2a8ad487 100644--- a/frontend/src/routes/+page.svelte+++ b/frontend/src/routes/+page.svelte@@ -8,7 +8,7 @@<div class="mb-4 flex w-64"><BiglogoWithText /></div>- <div class="w-full md:mx-0 md:max-w-2xl">+ <div class="w-full md:mx-0 md:max-w-xl"><Searchbar autofocus={true} /></div><div class="mt-3 flex space-x-2 text-neutral">diff --git a/frontend/src/routes/explore/+page.svelte b/frontend/src/routes/explore/+page.svelteindex 57e20c71..2b7ca5a4 100644--- a/frontend/src/routes/explore/+page.svelte+++ b/frontend/src/routes/explore/+page.svelte@@ -71,7 +71,7 @@</div><formclass={twJoin(- 'mb-2 flex w-full max-w-lg rounded-full border border-base-400 bg-base-100 p-[2px] pl-3 transition focus-within:shadow',+ 'mb-2 flex w-full max-w-lg rounded-full border border-base-400 bg-base-100 p-[1px] pl-3 transition focus-within:shadow',)}id="site-input-container"on:submit|preventDefault={() => addWebsite(inputWebsite, true)}diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelteindex 66d2d873..f690696c 100644--- a/frontend/src/routes/search/+page.svelte+++ b/frontend/src/routes/search/+page.svelte@@ -146,7 +146,7 @@class="m-0 grid w-full gap-y-6 px-5 pt-4 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] md:gap-x-12 md:pl-20 lg:px-36"style="text-rendering:optimizeLegibility;font-smooth:always;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;">- <div class="flex max-w-2xl flex-col space-y-5">+ <div class="flex max-w-2xl flex-col space-y-3"><div class="w-full"><Searchbar {query} /></div>diff --git a/frontend/static/robots.txt b/frontend/static/robots.txtindex e69de29b..4976c99f 100644--- a/frontend/static/robots.txt+++ b/frontend/static/robots.txt@@ -0,0 +1,11 @@+User-agent: AhrefsBot+Disallow: /++User-agent: SemrushBot+Disallow: /++User-agent: dotbot+Disallow: /++User-agent: linkdexbot+Disallow: /diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pynew file mode 100644index 00000000..fef78a75--- /dev/null+++ b/ltr/auto_annotate.py@@ -0,0 +1,199 @@+from openai import OpenAI+from os import environ+import re+import sqlite3+import requests+from tqdm import tqdm+import json+import time++NUM_RESULTS_PER_QUERY = 10+PROMPT = """You are an expert data annotator employed at Google. Your task is to evaluate search results on an integer scale of 0-4, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source or niche blog and answer the users query.+First briefly provide the reasoning for the evaluation and then give the relevancy on the form "Relevancy: {{}}" on a new line.++Query: "{}"+URL: "{}"+Title: "{}"+Snippet: "{}"+Explanation:"""++client = OpenAI(+ api_key=environ.get("OPENAI_API_KEY"),+)+++with open("data/queries_us.csv") as f:+ all_queries = [line.strip() for line in f.readlines()]+++def setup_db():+ db = sqlite3.connect("data/auto-ranking-annotation.sqlite")++ cur = db.cursor()++ cur.execute(+ """+ CREATE TABLE IF NOT EXISTS queries (+ qid UUID PRIMARY KEY DEFAULT (HEX(RANDOMBLOB(16))),+ query TEXT NOT NULL UNIQUE+ );"""+ )++ cur.execute(+ """+ CREATE TABLE IF NOT EXISTS search_results (+ qid UUID NOT NULL,+ url TEXT NOT NULL,+ orig_rank INTEGER NOT NULL,+ webpage_json TEXT NOT NULL,+ annotation INTEGER,+ PRIMARY KEY (qid, url)+ );"""+ )++ db.commit()++ return db+++db = setup_db()++cur = db.cursor()+for query in all_queries:+ cur.execute(+ """+ INSERT OR IGNORE INTO queries (query)+ VALUES (?)+ """,+ (query,),+ )++db.commit()++unannotated_queries = cur.execute(+ """+ SELECT qid, query+ FROM queries+ WHERE NOT EXISTS (+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL+ )+ ORDER BY qid+ """,+).fetchall()++unannotated_queries = {qid: query for qid, query in unannotated_queries}+++def simplify_snippet(snippet):+ if "text" not in snippet:+ return ""++ return "".join([f["text"] for f in snippet["text"]["fragments"]])+++def get_search_results(query):+ url = "https://trystract.com/beta/api/search"++ payload = {+ "query": query,+ "numResults": NUM_RESULTS_PER_QUERY,+ "returnRankingSignals": True,+ }++ return [+ {+ "url": w["url"],+ "title": w["title"],+ "rankingSignals": {s: v["value"] for (s, v) in w["rankingSignals"].items()},+ "snippet": simplify_snippet(w["snippet"]),+ }+ for w in requests.post(url, json=payload).json()["webpages"][+ :NUM_RESULTS_PER_QUERY+ ]+ ]+++# add search results to db+print("Adding search results to db")+for qid, query in tqdm(unannotated_queries.items()):+ has_results = (+ cur.execute("SELECT 1 FROM search_results WHERE qid = ?", (qid,)).fetchone()+ is not None+ )+ if has_results:+ continue++ results = get_search_results(query)+ time.sleep(1)++ for i, result in enumerate(results):+ cur.execute(+ """+ INSERT INTO search_results (qid, url, orig_rank, webpage_json)+ VALUES (?, ?, ?, ?)+ """,+ (qid, result["url"], i, json.dumps(result)),+ )++ db.commit()+++def get_prompt(query, url, title, snippet):+ return PROMPT.format(+ query,+ url,+ title,+ snippet,+ )+++def get_relevancy(res):+ regex = r"Relevancy: (\d)"+ matches = re.findall(regex, res)+ if len(matches) == 0:+ return None+ return int(matches[0])+++for qid, query in tqdm(unannotated_queries.items()):+ unnanotated_results = cur.execute(+ """+ SELECT url, orig_rank, webpage_json+ FROM search_results+ WHERE qid = ? AND annotation IS NULL+ ORDER BY orig_rank+ """,+ (qid,),+ ).fetchall()++ for url, orig_rank, webpage_json in tqdm(unnanotated_results):+ webpage = json.loads(webpage_json)+ prompt = get_prompt(query, url, webpage["title"], webpage["snippet"])+ res = (+ client.chat.completions.create(+ messages=[+ {+ "role": "system",+ "content": prompt,+ }+ ],+ model="gpt-4-1106-preview",+ )+ .choices[0]+ .message.content+ )++ relevancy = get_relevancy(res)+ if relevancy is None:+ continue++ cur.execute(+ """+ UPDATE search_results+ SET annotation = ?+ WHERE qid = ? AND url = ?+ """,+ (relevancy, qid, url),+ )++ db.commit()diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex d19915d3..3c856b28 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -3,9 +3,8 @@ import numpy as npimport jsonfrom pprint import pprintimport sqlite3-import sys-con = sqlite3.connect("data/ranking-annotation.sqlite")+con = sqlite3.connect("data/auto-ranking-annotation.sqlite")cur = con.cursor()res = cur.execute(@@ -129,13 +128,13 @@ model.booster_.save_model()# print feature importance-# pprint(-# sorted(-# [(id2feature[i], v) for i, v in enumerate(model.feature_importances_) if v > 0],-# key=lambda x: x[1],-# reverse=True,-# )-# )+pprint(+ sorted(+ [(id2feature[i], v) for i, v in enumerate(model.feature_importances_) if v > 0],+ key=lambda x: x[1],+ reverse=True,+ )+)# verify that the saved model outputs the same scores# for the same input