Skip to content

web search done right

Safe search (#79)

Mikkel Denker24ffa32983ad
diff --git a/.gitattributes b/.gitattributes
index a0b096af..43103bf0 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1,2 @@
core/testcases/** linguist-vendored
+scripts/** linguist-vendored
diff --git a/Cargo.lock b/Cargo.lock
index 760315ae..05d95633 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -115,16 +115,15 @@ dependencies = [
[[package]]
name = "anstream"
-version = "0.3.2"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
- "is-terminal",
"utf8parse",
]
@@ -154,9 +153,9 @@ dependencies = [
[[package]]
name = "anstyle-wincon"
-version = "1.0.2"
+version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
+checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd"
dependencies = [
"anstyle",
"windows-sys 0.48.0",
@@ -803,24 +802,24 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.3.24"
+version = "4.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb690e81c7840c0d7aade59f242ea3b41b9bc27bcd5997890e7702ae4b32e487"
+checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d"
dependencies = [
"clap_builder",
- "clap_derive 4.3.12",
+ "clap_derive 4.4.0",
"once_cell",
]
[[package]]
name = "clap_builder"
-version = "4.3.24"
+version = "4.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ed2e96bc16d8d740f6f48d663eddf4b8a0983e79210fd55479b7bcd0a69860e"
+checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6"
dependencies = [
"anstream",
"anstyle",
- "clap_lex 0.5.0",
+ "clap_lex 0.5.1",
"strsim",
]
@@ -839,9 +838,9 @@ dependencies = [
[[package]]
name = "clap_derive"
-version = "4.3.12"
+version = "4.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050"
+checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a"
dependencies = [
"heck 0.4.1",
"proc-macro2 1.0.66",
@@ -860,9 +859,9 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "0.5.0"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
[[package]]
name = "color_quant"
@@ -1781,6 +1780,7 @@ dependencies = [
"ahash 0.8.3",
"allocator-api2",
"rkyv",
+ "serde",
]
[[package]]
@@ -3571,9 +3571,9 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.11.19"
+version = "0.11.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3"
+checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
dependencies = [
"base64 0.21.2",
"bytes",
@@ -3992,9 +3992,9 @@ checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
[[package]]
name = "serde"
-version = "1.0.185"
+version = "1.0.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31"
+checksum = "9f5db24220c009de9bd45e69fb2938f4b6d2df856aa9304ce377b3180f83b7c1"
dependencies = [
"serde_derive",
]
@@ -4011,9 +4011,9 @@ dependencies = [
[[package]]
name = "serde_derive"
-version = "1.0.185"
+version = "1.0.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec"
+checksum = "5ad697f7e0b65af4983a4ce8f56ed5b357e8d3c36651bf6a7e13639c17b8e670"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.33",
@@ -4740,7 +4740,7 @@ checksum = "aea68938177975ab09da68552b720eac941779ff386baceaf77e0f5f9cea645f"
dependencies = [
"aho-corasick 0.7.20",
"cached-path",
- "clap 4.3.24",
+ "clap 4.4.0",
"derive_builder",
"dirs",
"esaxx-rs",
@@ -5532,9 +5532,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "winnow"
-version = "0.5.14"
+version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97"
+checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc"
dependencies = [
"memchr",
]
diff --git a/Cargo.toml b/Cargo.toml
index be2136c2..d7bea3a8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -97,6 +97,6 @@ utoipa = { version = "3.4.4", features = ["axum_extras"] }
utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }
tikv-jemallocator = "0.5"
publicsuffix = "2.2.3"
-hashbrown = {version = "0.14.0", features = ["rkyv"] }
+hashbrown = { version = "0.14.0", features = ["serde", "rkyv"] }
rkyv = { version = "0.7.42", default_features = false, features = ["std", "validation", "size_64"] }
diff --git a/core/Cargo.toml b/core/Cargo.toml
index ad2fbb34..680c9a85 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -143,4 +143,8 @@ harness = false
[[bench]]
name = "hyperloglog"
+harness = false
+
+[[bench]]
+name = "naive-bayes"
harness = false
\ No newline at end of file
diff --git a/core/benches/naive-bayes.rs b/core/benches/naive-bayes.rs
new file mode 100644
index 00000000..7d55c7b2
--- /dev/null
+++ b/core/benches/naive-bayes.rs
@@ -0,0 +1,76 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use rand::seq::SliceRandom;
+use stract::naive_bayes;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+enum Label {
+ Spam,
+ Ham,
+}
+
+impl naive_bayes::Label for Label {}
+
+fn spam_string(length: usize) -> String {
+ let mut res = String::with_capacity(length);
+ for _ in 0..length {
+ res.push_str("aaa");
+ res.push(' ');
+ }
+ res
+}
+
+fn ham_string(length: usize) -> String {
+ let mut res = String::with_capacity(length);
+ for _ in 0..length {
+ res.push_str("bbb");
+ res.push(' ');
+ }
+ res
+}
+
+fn create_dataset() -> Vec<(String, Label)> {
+ let mut res = Vec::new();
+
+ for _ in 0..1000 {
+ res.push((spam_string(100), Label::Spam));
+ res.push((ham_string(100), Label::Ham));
+ }
+ res.shuffle(&mut rand::thread_rng());
+
+ res
+}
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+ let dataset = create_dataset();
+
+ // c.bench_function("train naive bayes classifier", |b| {
+ // b.iter(|| {
+ // for _ in 0..1000 {
+ // let mut pipeline = naive_bayes::Pipeline::new();
+ // pipeline.fit(&dataset);
+ // }
+ // })
+ // });
+
+ let mut pipeline = naive_bayes::Pipeline::new();
+ pipeline.fit(&dataset);
+ let text = spam_string(10_000);
+
+ let num_samples = 100_000;
+ c.bench_function("predict", |b| {
+ b.iter(|| {
+ let start = std::time::Instant::now();
+ for _ in 0..num_samples {
+ pipeline.predict(&text);
+ }
+
+ println!(
+ "{:.2} ms/pred",
+ start.elapsed().as_millis() as f32 / num_samples as f32
+ );
+ })
+ });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index f25a28ad..f28c40af 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -171,6 +171,7 @@ impl Searcher {
site_rankings: None,
return_ranking_signals: false,
flatten_response: false,
+ safe_search: Some(false),
};
tracing::debug!("searching at {:?}: {:#?}", self.url, query);
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 5dd331bc..a0e4dd15 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -65,6 +65,7 @@ mod index;
mod metrics;
mod opensearch;
mod optics;
+mod preferences;
mod privacy;
pub mod search;
mod sites;
@@ -192,7 +193,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route
.route("/explore/export", get(explore::export))
.route("/chat", get(chat::route))
.route("/about", get(about::route))
- .route("/settings", get(optics::route))
+ .route("/settings", get(preferences::route))
.route("/settings/optics", get(optics::route))
.route("/settings/sites", get(sites::route))
.route("/settings/sites/export", get(sites::export))
diff --git a/core/src/api/optics.rs b/core/src/api/optics.rs
index 9732f31a..00d625f5 100644
--- a/core/src/api/optics.rs
+++ b/core/src/api/optics.rs
@@ -76,7 +76,7 @@ pub async fn route(extract::State(state): extract::State<Arc<State>>) -> impl In
}
#[derive(Template)]
-#[template(path = "settings/index.html")]
+#[template(path = "settings/optics/index.html")]
struct OpticsTemplate {
default_optics: Vec<OpticLink>,
with_alice: Option<bool>,
diff --git a/core/src/api/preferences.rs b/core/src/api/preferences.rs
new file mode 100644
index 00000000..ca64d22e
--- /dev/null
+++ b/core/src/api/preferences.rs
@@ -0,0 +1,34 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::sync::Arc;
+
+use super::{HtmlTemplate, State};
+use askama::Template;
+use axum::{extract, response::IntoResponse};
+
+#[allow(clippy::unused_async)]
+pub async fn route(extract::State(state): extract::State<Arc<State>>) -> impl IntoResponse {
+ let template = PreferencesTemplate {
+ with_alice: state.config.with_alice,
+ };
+ HtmlTemplate(template)
+}
+
+#[derive(Template)]
+#[template(path = "settings/index.html")]
+struct PreferencesTemplate {
+ with_alice: Option<bool>,
+}
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index 8cd1aa22..98b0357e 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -97,11 +97,17 @@ fn extract_site_rankings(params: &SearchParams) -> Option<SiteRankings> {
#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)]
pub struct SearchParams {
+ /// Query
pub q: Option<String>,
+ /// Page
pub p: Option<usize>,
+ /// Language
pub gl: Option<String>,
pub optic: Option<String>,
+ /// Site rankings
pub sr: Option<String>,
+ /// Safe search
+ pub ss: Option<bool>,
}
#[allow(clippy::unused_async)]
@@ -166,6 +172,7 @@ pub async fn route(
site_rankings,
num_results: NUM_RESULTS_PER_PAGE,
return_ranking_signals: false,
+ safe_search: params.ss.unwrap_or(defaults::SearchQuery::safe_search()),
})
.await
{
@@ -287,6 +294,7 @@ pub struct ApiSearchQuery {
pub selected_region: Option<Region>,
pub optic: Option<String>,
pub site_rankings: Option<SiteRankings>,
+ pub safe_search: Option<bool>,
#[serde(default = "defaults::SearchQuery::return_ranking_signals")]
pub return_ranking_signals: bool,
@@ -315,6 +323,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {
optic,
site_rankings: api.site_rankings,
return_ranking_signals: api.return_ranking_signals,
+ safe_search: api.safe_search.unwrap_or(default.safe_search),
})
}
}
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index e7b6ee50..2447a569 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -78,7 +78,7 @@ impl Crawler {
}
pub fn min_crawl_delay_ms() -> u64 {
- 4_000
+ 1_000
}
pub fn max_crawl_delay_ms() -> u64 {
@@ -124,4 +124,8 @@ impl SearchQuery {
pub fn return_ranking_signals() -> bool {
false
}
+
+ pub fn safe_search() -> bool {
+ false
+ }
}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 66096a4b..acaff635 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -35,6 +35,7 @@ pub struct IndexingLocalConfig {
pub topics_path: Option<String>,
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
+ pub safety_classifier_path: Option<String>,
pub minimum_clean_words: Option<usize>,
}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 71abc93b..4ce6c1ee 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -139,6 +139,7 @@ fn create_inverted_index() -> Result<()> {
.unwrap()
.to_string(),
),
+ None,
);
let index = indexer::process_job(&job, &worker);
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 76ca799f..26ce51e3 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -30,7 +30,7 @@ use crate::mapreduce::{Map, Reduce, Worker};
use crate::ranking::centrality_store::IndexerCentralityStore;
use crate::ranking::SignalAggregator;
use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
-use crate::webpage::{Html, Webpage};
+use crate::webpage::{safety_classifier, Html, Webpage};
use crate::{human_website_annotations, Result};
pub struct Indexer {}
@@ -76,6 +76,7 @@ pub struct IndexingWorker {
page_centrality_store: Option<IndexerCentralityStore>,
webgraph: Option<Webgraph>,
topics: Option<human_website_annotations::Mapper>,
+ safety_classifier: Option<safety_classifier::Model>,
}
impl IndexingWorker {
@@ -84,12 +85,15 @@ impl IndexingWorker {
page_centrality_store_path: Option<String>,
webgraph_path: Option<String>,
topics_path: Option<String>,
+ safety_classifier_path: Option<String>,
) -> Self {
Self {
host_centrality_store: IndexerCentralityStore::open(host_centrality_store_path),
page_centrality_store: page_centrality_store_path.map(IndexerCentralityStore::open),
webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).single_threaded().open()),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
+ safety_classifier: safety_classifier_path
+ .map(|path| safety_classifier::Model::open(path).unwrap()),
}
}
}
@@ -230,8 +234,13 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
pre_computed_score: 0.0,
node_id: Some(host_node_id),
dmoz_description,
+ safety_classification: None,
};
+ if let Some(model) = worker.safety_classifier.as_ref() {
+ webpage.safety_classification = Some(model.predict(&webpage).label);
+ }
+
let mut signal_aggregator = SignalAggregator::new(None);
signal_aggregator.set_current_timestamp(current_timestamp);
@@ -327,6 +336,7 @@ impl Indexer {
config.page_centrality_store_path.clone(),
config.page_webgraph_path.clone(),
config.topics_path.clone(),
+ config.safety_classifier_path.clone(),
);
let indexes = warc_paths
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index e33a4725..1b3a8308 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -25,6 +25,7 @@ pub mod dmoz_parser;
mod entity;
pub mod frontend;
pub mod indexer;
+pub mod safety_classifier;
pub mod search_server;
mod webgraph;
pub mod webgraph_server;
diff --git a/core/src/entrypoint/safety_classifier.rs b/core/src/entrypoint/safety_classifier.rs
new file mode 100644
index 00000000..e3ddd37d
--- /dev/null
+++ b/core/src/entrypoint/safety_classifier.rs
@@ -0,0 +1,65 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rand::seq::SliceRandom;
+use tracing::info;
+
+use crate::{webpage, Result};
+use std::path::Path;
+
+const TEST_SIZE: f64 = 0.2;
+
+pub fn train<P: AsRef<Path>>(dataset: P, output: P) -> Result<()> {
+ if !dataset.as_ref().exists() {
+ return Err(anyhow::anyhow!(
+ "dataset path {:?} does not exist",
+ dataset.as_ref()
+ ));
+ }
+
+ let mut model = webpage::safety_classifier::Model::new();
+ let mut dataset = webpage::safety_classifier::load_dataset(dataset)?;
+
+ if dataset.is_empty() {
+ return Err(anyhow::anyhow!("dataset is empty"));
+ }
+
+ dataset.shuffle(&mut rand::thread_rng());
+
+ let test_size = (dataset.len() as f64 * TEST_SIZE) as usize;
+ let test_set = dataset.split_off(dataset.len() - test_size);
+
+ model.fit(&dataset);
+ let evaluation = model.evaluate(&test_set);
+
+ info!("accuracy: {}", evaluation.accuracy);
+ info!("precision: {}", evaluation.precision);
+ info!("recall: {}", evaluation.recall);
+ info!("f1: {}", evaluation.f1);
+
+ model.save(output)?;
+
+ Ok(())
+}
+
+pub fn predict<P: AsRef<Path>>(model: P, text: &str) -> Result<()> {
+ let model = webpage::safety_classifier::Model::open(model)?;
+ let pred = model.predict_text(text);
+
+ info!("{:#?}", pred);
+
+ Ok(())
+}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index a40346e2..959f1ce5 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -907,6 +907,7 @@ mod tests {
pre_computed_score: 0.0,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 296640c7..b2e94909 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -58,6 +58,7 @@ mod kv;
mod leaky_queue;
mod llm_utils;
mod metrics;
+pub mod naive_bayes;
pub mod prehashed;
mod qa_model;
mod query;
diff --git a/core/src/main.rs b/core/src/main.rs
index a2bfba79..cc861717 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -25,7 +25,7 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};
use stract::entrypoint::configure;
use stract::entrypoint::indexer::IndexPointer;
-use stract::entrypoint::{self, frontend, search_server, webgraph_server};
+use stract::entrypoint::{self, frontend, safety_classifier, search_server, webgraph_server};
use stract::webgraph::WebgraphBuilder;
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -103,6 +103,12 @@ enum Commands {
/// and sending newly discovered urls back to the crawl coordinator.
Crawler { config_path: String },
+ /// Train or run inference on the classifier that predicts if a webpage is NSFW or SFW.
+ SafetyClassifier {
+ #[clap(subcommand)]
+ options: SafetyClassifierOptions,
+ },
+
/// Setup dev environment.
#[cfg(feature = "dev")]
Configure {
@@ -123,6 +129,19 @@ enum AliceOptions {
GenerateKey,
}
+/// Commands to train or run inference on the classifier that predicts if a webpage is NSFW or SFW.
+#[derive(Subcommand)]
+enum SafetyClassifierOptions {
+ /// Train the classifier
+ Train {
+ dataset_path: String,
+ output_path: String,
+ },
+
+ /// Run a single prediction to test the model
+ Predict { model_path: String, text: String },
+}
+
#[derive(Subcommand)]
enum CentralityType {
/// Calculate all the supported centrality metrics.
@@ -311,6 +330,15 @@ fn main() -> Result<()> {
.build()?
.block_on(entrypoint::crawler::worker(config))?
}
+ Commands::SafetyClassifier { options } => match options {
+ SafetyClassifierOptions::Train {
+ dataset_path,
+ output_path,
+ } => safety_classifier::train(dataset_path, output_path)?,
+ SafetyClassifierOptions::Predict { model_path, text } => {
+ safety_classifier::predict(model_path, &text)?
+ }
+ },
}
Ok(())
diff --git a/core/src/naive_bayes.rs b/core/src/naive_bayes.rs
new file mode 100644
index 00000000..729019ba
--- /dev/null
+++ b/core/src/naive_bayes.rs
@@ -0,0 +1,372 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::intmap::IntMap;
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
+use std::hash::Hash;
+
+/// Laplace smoothing factor
+const ALPHA: f32 = 1.0;
+
+/// Struct for sparse Term Frequency-Inverse Document Frequency (TF-IDF) features.
+#[derive(Debug)]
+pub struct TfIdf {
+ pub term_id: u64,
+ pub value: f32,
+}
+
+/// Vectorizer to convert text into TF-IDF features
+#[derive(serde::Deserialize, serde::Serialize)]
+struct TfidfVectorizer {
+ vocabulary: HashMap<String, u64>,
+ idf: IntMap<u64, f32>,
+}
+
+impl TfidfVectorizer {
+ pub fn new() -> Self {
+ Self {
+ vocabulary: HashMap::new(),
+ idf: IntMap::new(),
+ }
+ }
+
+ pub fn fit(&mut self, corpus: &[String]) {
+ let doc_freq = self.calculate_document_frequencies(corpus);
+ self.build_vocabulary_and_idf(corpus, &doc_freq);
+ }
+
+ /// Transforms a document into TF-IDF features
+ pub fn transform(&self, doc: &str) -> Vec<TfIdf> {
+ let term_frequencies = self.calculate_term_frequencies(doc);
+ self.calculate_tfidf(&term_frequencies)
+ }
+
+ /// Helper function to calculate document frequencies
+ fn calculate_document_frequencies(&self, corpus: &[String]) -> HashMap<String, usize> {
+ let mut doc_freq: HashMap<String, usize> = HashMap::new();
+ let mut doc_words: HashSet<&str> = HashSet::new();
+
+ for doc in corpus {
+ doc_words.clear();
+ for word in doc.split_whitespace() {
+ if doc_words.insert(word) {
+ *doc_freq.entry(word.to_string()).or_insert(0) += 1;
+ }
+ }
+ }
+ doc_freq
+ }
+
+ /// Helper function to build vocabulary and idf
+ fn build_vocabulary_and_idf(&mut self, corpus: &[String], doc_freq: &HashMap<String, usize>) {
+ let mut vocab: Vec<String> = doc_freq.keys().cloned().collect();
+ vocab.sort();
+ for (term_id, word) in vocab.iter().enumerate() {
+ let term_id = term_id as u64;
+ self.vocabulary.insert(word.to_owned(), term_id);
+ self.idf.insert(
+ term_id,
+ ((corpus.len() as f32) / (1.0 + doc_freq[word] as f32)).ln() + 1.0,
+ );
+ }
+ }
+
+ /// Helper function to calculate term frequencies for a document
+ fn calculate_term_frequencies(&self, doc: &str) -> IntMap<u64, f32> {
+ let mut tf: IntMap<u64, f32> = IntMap::new();
+ for word in doc.split_whitespace() {
+ if let Some(&term_id) = self.vocabulary.get(word) {
+ let c = tf.get(&term_id).copied().unwrap_or_default();
+ tf.insert(term_id, c + 1.0);
+ }
+ }
+ tf
+ }
+
+ /// Helper function to calculate TF-IDF values
+ fn calculate_tfidf(&self, term_frequencies: &IntMap<u64, f32>) -> Vec<TfIdf> {
+ term_frequencies
+ .iter()
+ .map(|(term_id, tf)| TfIdf {
+ term_id: *term_id,
+ value: tf * self.idf.get(term_id).copied().unwrap_or_default(),
+ })
+ .collect()
+ }
+}
+
+pub trait Label: Hash + PartialEq + Eq + Clone + PartialOrd + Ord {}
+
+impl Label for String {}
+
+/// Struct to represent a data point for classification
+#[derive(Debug)]
+pub struct Datapoint<L: Label> {
+ pub features: Vec<TfIdf>,
+ pub label: L,
+}
+
+#[derive(Debug)]
+pub struct Prediction<L> {
+ pub label: L,
+ pub confidence: f32,
+}
+
+/// Naive Bayes Classifier
+#[derive(serde::Deserialize, serde::Serialize)]
+pub struct NaiveBayes<L: Label> {
+ classes: Vec<L>,
+ class_prior: Vec<f32>,
+ feature_log_prob: Vec<IntMap<u64, f32>>,
+}
+
+impl Default for NaiveBayes<String> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<L: Label> NaiveBayes<L> {
+ pub fn new() -> Self {
+ Self {
+ classes: Vec::new(),
+ class_prior: Vec::new(),
+ feature_log_prob: Vec::new(),
+ }
+ }
+
+ pub fn fit(&mut self, datapoints: &[Datapoint<L>]) {
+ if datapoints.is_empty() {
+ return;
+ }
+ self.classes = self.extract_unique_classes(datapoints);
+ let class_counts = self.count_classes(datapoints);
+ self.calculate_feature_log_prob(&class_counts, datapoints);
+ }
+
+ /// Predicts the class for a given sample
+ pub fn predict(&self, sample: &[TfIdf]) -> Prediction<L> {
+ let class_log_probs = self.calculate_class_log_probs(sample);
+
+ let mut best_class_index = 0;
+ let mut best_class_log_prob = f32::INFINITY;
+
+ let mut s = 0.0;
+
+ for (class_id, class_log_prob) in class_log_probs {
+ if class_log_prob < best_class_log_prob {
+ best_class_index = class_id;
+ best_class_log_prob = class_log_prob;
+ }
+ s += class_log_prob;
+ }
+
+ let label = self.classes[best_class_index].clone();
+ let confidence = (best_class_log_prob / s).max(0.0);
+ Prediction { label, confidence }
+ }
+
+ /// Helper function to extract unique classes from the data points
+ fn extract_unique_classes(&self, datapoints: &[Datapoint<L>]) -> Vec<L> {
+ datapoints
+ .iter()
+ .map(|d| &d.label)
+ .unique()
+ .cloned()
+ .sorted()
+ .collect()
+ }
+
+ /// Helper function to count the number of instances for each class
+ fn count_classes(&self, datapoints: &[Datapoint<L>]) -> HashMap<L, usize> {
+ let mut class_counts: HashMap<L, usize> = HashMap::new();
+ for datapoint in datapoints {
+ *class_counts.entry(datapoint.label.clone()).or_insert(0) += 1;
+ }
+ class_counts
+ }
+
+ /// Helper function to calculate feature log probabilities for each class
+ fn calculate_feature_log_prob(
+ &mut self,
+ class_counts: &HashMap<L, usize>,
+ datapoints: &[Datapoint<L>],
+ ) {
+ let num_classes = self.classes.len();
+ let mut feature_log_prob = vec![IntMap::new(); num_classes];
+ for (class_id, class) in self.classes.iter().enumerate() {
+ let mut counts: IntMap<u64, f32> = IntMap::new();
+ let mut total_count: f32 = 0.0;
+ for datapoint in datapoints {
+ if &datapoint.label == class {
+ for feature in &datapoint.features {
+ let count = counts.get(&feature.term_id).copied().unwrap_or_default();
+ counts.insert(feature.term_id, count + feature.value);
+ total_count += feature.value;
+ }
+ }
+ }
+ for (term_id, count) in counts.iter() {
+ let log_prob = ((count + ALPHA) / (total_count + ALPHA)).ln();
+ feature_log_prob[class_id].insert(*term_id, log_prob);
+ }
+ }
+ self.class_prior = self
+ .classes
+ .iter()
+ .map(|class| class_counts[class] as f32 / datapoints.len() as f32)
+ .collect();
+ self.feature_log_prob = feature_log_prob;
+ }
+
+ /// Helper function to calculate the log probabilities for each class
+ fn calculate_class_log_probs<'a>(
+ &'a self,
+ sample: &'a [TfIdf],
+ ) -> impl Iterator<Item = (usize, f32)> + 'a {
+ self.classes
+ .iter()
+ .enumerate()
+ .map(|(id, _)| id)
+ .map(move |class_id| {
+ let mut log_prob = self.class_prior[class_id].ln();
+ for feature in sample {
+ if feature.value == 0.0 {
+ continue;
+ }
+
+ log_prob += feature.value
+ * self.feature_log_prob[class_id]
+ .get(&feature.term_id)
+ .copied()
+ .unwrap_or(ALPHA / (ALPHA + ALPHA)) // Laplace smoothing
+ }
+ (class_id, log_prob)
+ })
+ }
+}
+
+#[derive(serde::Deserialize, serde::Serialize)]
+pub struct Pipeline<L: Label> {
+ vectorizer: TfidfVectorizer,
+ classifier: NaiveBayes<L>,
+}
+
+impl Default for Pipeline<String> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<L: Label> Pipeline<L> {
+ pub fn new() -> Self {
+ Self {
+ vectorizer: TfidfVectorizer::new(),
+ classifier: NaiveBayes::new(),
+ }
+ }
+
+ pub fn fit(&mut self, data: &[(String, L)]) {
+ let (corpus, labels): (Vec<_>, Vec<_>) = data.iter().cloned().unzip();
+
+ self.vectorizer.fit(&corpus);
+ let datapoints: Vec<Datapoint<_>> = corpus
+ .iter()
+ .zip_eq(labels)
+ .map(|(doc, label)| Datapoint {
+ features: self.vectorizer.transform(doc),
+ label: label.clone(),
+ })
+ .collect();
+ self.classifier.fit(&datapoints);
+ }
+
+ pub fn predict(&self, doc: &str) -> Prediction<L> {
+ let features = self.vectorizer.transform(doc);
+ self.classifier.predict(&features)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_naive_bayes() {
+ let x = vec![
+ vec![1.0, 1.0, 0.0],
+ vec![1.0, 1.0, 0.0],
+ vec![1.0, 1.0, 0.0],
+ vec![1.0, 1.0, 0.0],
+ vec![1.0, 1.0, 0.0],
+ vec![0.0, 1.0, 1.0],
+ vec![0.0, 1.0, 1.0],
+ vec![0.0, 1.0, 1.0],
+ vec![0.0, 1.0, 1.0],
+ vec![0.0, 1.0, 1.0],
+ ];
+ let y = vec![
+ "ham".to_owned(),
+ "ham".to_owned(),
+ "ham".to_owned(),
+ "ham".to_owned(),
+ "ham".to_owned(),
+ "spam".to_owned(),
+ "spam".to_owned(),
+ "spam".to_owned(),
+ "spam".to_owned(),
+ "spam".to_owned(),
+ ];
+
+ let datapoints: Vec<Datapoint<_>> = x
+ .into_iter()
+ .zip_eq(y)
+ .map(|(x, y)| {
+ let features = x
+ .into_iter()
+ .enumerate()
+ .filter_map(|(i, v)| {
+ if v == 0.0 {
+ None
+ } else {
+ Some(TfIdf {
+ term_id: i as u64,
+ value: v,
+ })
+ }
+ })
+ .collect();
+ Datapoint { features, label: y }
+ })
+ .collect();
+
+ let mut model = NaiveBayes::new();
+ model.fit(&datapoints);
+ let pred = model.predict(&[TfIdf {
+ term_id: 0,
+ value: 1.0,
+ }]);
+ assert_eq!(pred.label, "ham".to_owned());
+
+ let pred = model.predict(&[TfIdf {
+ term_id: 2,
+ value: 1.0,
+ }]);
+
+ assert_eq!(pred.label, "spam".to_owned());
+ }
+}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 6ce4180d..1f2c5910 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -15,12 +15,17 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- inverted_index::InvertedIndex, ranking::SignalCoefficient, search_ctx::Ctx,
- searcher::SearchQuery, webpage::region::Region, Result,
+ inverted_index::InvertedIndex,
+ ranking::SignalCoefficient,
+ schema::{Field, TextField},
+ search_ctx::Ctx,
+ searcher::SearchQuery,
+ webpage::{region::Region, safety_classifier},
+ Result,
};
use optics::{Optic, SiteRankings};
use std::collections::HashMap;
-use tantivy::query::{BooleanQuery, Occur, QueryClone};
+use tantivy::query::{BooleanQuery, Occur, QueryClone, TermQuery};
mod const_query;
pub mod intersection;
@@ -71,11 +76,29 @@ impl Query {
let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =
schema.fields().collect();
- let queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
+ let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
.map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
.collect();
+ if query.safe_search {
+ let field = Field::Text(TextField::SafetyClassification);
+ let field = schema.get_field(field.name()).unwrap();
+
+ queries.push((
+ Occur::MustNot,
+ Box::new(TermQuery::new(
+ tantivy::Term::from_field_text(
+ field,
+ safety_classifier::Label::NSFW.to_string().as_str(),
+ ),
+ tantivy::schema::IndexRecordOption::Basic,
+ )),
+ ));
+ }
+
+ let mut tantivy_query = Box::new(BooleanQuery::new(queries));
+
let simple_terms_text: Vec<String> = terms
.clone()
.into_iter()
@@ -87,8 +110,6 @@ impl Query {
})
.collect();
- let mut tantivy_query = Box::new(BooleanQuery::new(queries));
-
let mut optics = Vec::new();
if let Some(site_rankigns_optic) = query.site_rankings.clone().map(|sr| sr.into_optic()) {
optics.push(site_rankigns_optic);
@@ -776,4 +797,77 @@ mod tests {
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
+
+ #[test]
+ fn safe_search() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let mut webpage = Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.sfw.com",
+ )
+ .unwrap();
+
+ webpage.safety_classification = Some(safety_classifier::Label::SFW);
+ webpage.html.set_clean_text("sfw".to_string());
+
+ index.insert(webpage).expect("failed to insert webpage");
+
+ let mut webpage = Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.nsfw.com",
+ )
+ .unwrap();
+
+ webpage.safety_classification = Some(safety_classifier::Label::NSFW);
+ webpage.html.set_clean_text("nsfw".to_string());
+
+ index.insert(webpage).expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ query: "test".to_string(),
+ safe_search: false,
+ ..Default::default()
+ };
+
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 2);
+
+ let query = SearchQuery {
+ query: "test".to_string(),
+ safe_search: true,
+ ..Default::default()
+ };
+
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+
+ assert_eq!(result.webpages[0].url, "https://www.sfw.com/");
+ }
}
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 98e1c78d..d8e4a903 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -277,6 +277,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -305,6 +306,7 @@ mod tests {
fetch_time_ms: 500,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -403,6 +405,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -430,6 +433,7 @@ mod tests {
fetch_time_ms: 500,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -506,6 +510,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -534,6 +539,7 @@ mod tests {
fetch_time_ms: 500,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -562,6 +568,7 @@ mod tests {
fetch_time_ms: 500,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -664,6 +671,7 @@ mod tests {
pre_computed_score: 0.0,
dmoz_description: None,
+ safety_classification: None,
node_id: Some(Node::from("www.a.com").into_host().id()),
})
.expect("failed to insert webpage");
@@ -692,6 +700,7 @@ mod tests {
pre_computed_score: 0.0,
fetch_time_ms: 500,
dmoz_description: None,
+ safety_classification: None,
node_id: Some(Node::from("www.b.com").into_host().id()),
})
.expect("failed to insert webpage");
@@ -720,6 +729,7 @@ mod tests {
pre_computed_score: 0.0,
fetch_time_ms: 500,
dmoz_description: None,
+ safety_classification: None,
node_id: Some(Node::from("www.c.com").into_host().id()),
})
.expect("failed to insert webpage");
@@ -796,6 +806,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -839,6 +850,7 @@ mod tests {
fetch_time_ms: 500,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -973,6 +985,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -1047,6 +1060,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -1076,6 +1090,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -1130,6 +1145,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1188,6 +1204,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1349,6 +1366,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1422,6 +1440,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1482,6 +1501,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -1511,6 +1531,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1609,6 +1630,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
};
page.html.set_clean_text("".to_string());
@@ -1688,6 +1710,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
};
page.html.set_clean_text("".to_string());
@@ -1716,6 +1739,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
};
page.html.set_clean_text("".to_string());
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 77784573..7172a908 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -372,6 +372,7 @@ mod tests {
node_id: Some(Node::from("e.com").id()),
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -401,6 +402,7 @@ mod tests {
node_id: Some(Node::from("d.com").id()),
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 042fb8f3..a5819207 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -176,6 +176,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -204,6 +205,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -251,6 +253,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -279,6 +282,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -325,6 +329,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -353,6 +358,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -395,6 +401,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -432,6 +439,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -475,6 +483,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -500,6 +509,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -543,6 +553,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -569,6 +580,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -596,6 +608,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
@@ -670,6 +683,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
};
w.html.set_clean_text(format!(
"{CONTENT} termA termB d d d d d d d d d {}",
@@ -702,6 +716,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
};
w.html.set_clean_text(format!(
"{CONTENT} termA d d d d d d d d d termB {}",
@@ -734,6 +749,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
};
w.html.set_clean_text(format!(
"{CONTENT} termA d d d d termB d d d d d {}",
@@ -788,6 +804,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -816,6 +833,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -865,6 +883,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -893,6 +912,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -921,6 +941,7 @@ mod tests {
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index a8ec071f..de0da903 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -104,6 +104,7 @@ mod tests {
dmoz_description: None,
node_id: Some(Node::from("https://www.first.com").into_host().id()),
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -133,6 +134,7 @@ mod tests {
dmoz_description: None,
node_id: Some(Node::from("https://www.second.com").into_host().id()),
+ safety_classification: None,
})
.expect("failed to insert webpage");
index
@@ -162,6 +164,7 @@ mod tests {
dmoz_description: None,
node_id: Some(Node::from("https://www.third.com").into_host().id()),
+ safety_classification: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 3df612a9..4f2c3229 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -52,6 +52,8 @@ pub enum TextField {
CleanBodyTrigrams,
TitleTrigrams,
MicroformatTags,
+ /// can either be NSFW or SFW (see safety classifier)
+ SafetyClassification,
}
impl From<TextField> for usize {
@@ -88,6 +90,7 @@ impl TextField {
TextField::CleanBodyTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),
TextField::TitleTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),
TextField::MicroformatTags => Tokenizer::default(),
+ TextField::SafetyClassification => Tokenizer::Identity(Identity {}),
}
}
@@ -126,6 +129,7 @@ impl TextField {
TextField::CleanBodyTrigrams => false,
TextField::TitleTrigrams => false,
TextField::MicroformatTags => true,
+ TextField::SafetyClassification => false,
}
}
@@ -156,6 +160,7 @@ impl TextField {
TextField::CleanBodyTrigrams => "clean_body_trigrams",
TextField::TitleTrigrams => "title_trigrams",
TextField::MicroformatTags => "microformat_tags",
+ TextField::SafetyClassification => "safety_classification",
}
}
}
@@ -245,7 +250,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 56] = [
+pub static ALL_FIELDS: [Field; 57] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -271,6 +276,7 @@ pub static ALL_FIELDS: [Field; 56] = [
Field::Text(TextField::CleanBodyTrigrams),
Field::Text(TextField::TitleTrigrams),
Field::Text(TextField::MicroformatTags),
+ Field::Text(TextField::SafetyClassification),
// FAST FIELDS
Field::Fast(FastField::IsHomepage),
Field::Fast(FastField::HostCentrality),
@@ -395,6 +401,9 @@ impl Field {
Field::Text(TextField::MicroformatTags) => {
IndexingOption::Text(self.default_text_options())
}
+ Field::Text(TextField::SafetyClassification) => {
+ IndexingOption::Text(self.default_text_options())
+ }
Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index a26b3906..a0ed050e 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -450,6 +450,7 @@ mod tests {
pre_computed_score: 0.0,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
.expect("failed to insert webpage");
}
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 83f21086..cc820982 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -67,6 +67,7 @@ pub struct SearchQuery {
pub optic: Option<Optic>,
pub site_rankings: Option<SiteRankings>,
pub return_ranking_signals: bool,
+ pub safe_search: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -91,6 +92,7 @@ impl Default for SearchQuery {
optic: Default::default(),
site_rankings: Default::default(),
return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),
+ safe_search: defaults::SearchQuery::safe_search(),
}
}
}
diff --git a/core/src/spell/spell_checker.rs b/core/src/spell/spell_checker.rs
index 8f86b646..57c4233b 100644
--- a/core/src/spell/spell_checker.rs
+++ b/core/src/spell/spell_checker.rs
@@ -37,7 +37,8 @@ fn deletes_rec(
if !word.is_empty() {
for (i, _) in word.char_indices() {
let delete = word
- .char_indices()
+ .chars()
+ .enumerate()
.filter(|(j, _)| *j != i)
.map(|(_, c)| c)
.collect::<String>();
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index fde2c31c..f03f16ad 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -33,6 +33,7 @@ use whatlang::Lang;
mod just_text;
pub mod region;
+pub mod safety_classifier;
pub mod schema_org;
pub mod url_ext;
@@ -121,6 +122,7 @@ pub struct Webpage {
pub pre_computed_score: f64,
pub node_id: Option<NodeID>,
pub dmoz_description: Option<String>,
+ pub safety_classification: Option<safety_classifier::Label>,
}
impl Webpage {
@@ -137,6 +139,7 @@ impl Webpage {
pre_computed_score: 0.0,
node_id: None,
dmoz_description: None,
+ safety_classification: None,
})
}
@@ -189,6 +192,18 @@ impl Webpage {
backlink_text,
);
+ let safety = self
+ .safety_classification
+ .map(|label| label.to_string())
+ .unwrap_or_default();
+
+ doc.add_text(
+ schema
+ .get_field(Field::Text(TextField::SafetyClassification).name())
+ .expect("Failed to get safety_classification field"),
+ safety,
+ );
+
doc.add_u64(
schema
.get_field(Field::Fast(FastField::HostCentrality).name())
@@ -1236,6 +1251,7 @@ impl Html {
doc.add_u64(tantivy_field, num_digits as u64);
}
Field::Text(TextField::BacklinkText)
+ | Field::Text(TextField::SafetyClassification)
| Field::Fast(FastField::HostCentrality)
| Field::Fast(FastField::PageCentrality)
| Field::Fast(FastField::FetchTimeMs)
@@ -2138,6 +2154,7 @@ mod tests {
pre_computed_score: 0.0,
node_id: None,
dmoz_description: Some("dmoz description".to_string()),
+ safety_classification: None,
};
assert_eq!(
@@ -2172,6 +2189,7 @@ mod tests {
pre_computed_score: 0.0,
node_id: None,
dmoz_description: Some("dmoz description".to_string()),
+ safety_classification: None,
};
assert_eq!(webpage.dmoz_description(), None)
diff --git a/core/src/webpage/safety_classifier.rs b/core/src/webpage/safety_classifier.rs
new file mode 100644
index 00000000..67c59b56
--- /dev/null
+++ b/core/src/webpage/safety_classifier.rs
@@ -0,0 +1,188 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::fs::OpenOptions;
+use std::path::Path;
+
+use itertools::Itertools;
+
+use crate::naive_bayes;
+use crate::Result;
+
+const MAX_NUM_WORDS: usize = 1000;
+
+#[derive(
+ Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
+)]
+pub enum Label {
+ SFW,
+ NSFW,
+}
+
+impl ToString for Label {
+ fn to_string(&self) -> String {
+ match self {
+ Label::SFW => "SFW".to_string(),
+ Label::NSFW => "NSFW".to_string(),
+ }
+ }
+}
+
+impl TryFrom<&str> for Label {
+ type Error = String;
+
+ fn try_from(value: &str) -> Result<Self, Self::Error> {
+ match value {
+ "SFW" => Ok(Label::SFW),
+ "NSFW" => Ok(Label::NSFW),
+ _ => Err(format!("invalid label: {}", value)),
+ }
+ }
+}
+
+impl naive_bayes::Label for Label {}
+
+#[derive(Debug, serde::Deserialize, serde::Serialize)]
+pub struct Datapoint {
+ pub label: Label,
+ pub text: String,
+}
+
+pub fn load_dataset<P: AsRef<Path>>(path: P) -> Result<Vec<Datapoint>> {
+ let mut datapoints = Vec::new();
+ let mut reader = csv::Reader::from_path(path)?;
+ for result in reader.deserialize() {
+ let datapoint: Datapoint = result?;
+ datapoints.push(datapoint);
+ }
+ Ok(datapoints)
+}
+
+fn normalize(text: &str) -> String {
+ text.split_whitespace()
+ .take(MAX_NUM_WORDS)
+ .join(" ")
+ .to_lowercase()
+}
+
+pub fn page_text(page: &crate::webpage::Webpage) -> String {
+ page.html.title().unwrap_or_default()
+ + " "
+ + page.html.clean_text().cloned().unwrap_or_default().as_str()
+}
+
+pub struct Evaluation {
+ pub accuracy: f64,
+ pub precision: f64,
+ pub recall: f64,
+ pub f1: f64,
+}
+
+#[derive(serde::Deserialize, serde::Serialize)]
+pub struct Model {
+ pipeline: naive_bayes::Pipeline<Label>,
+}
+
+impl Default for Model {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl Model {
+ pub fn new() -> Self {
+ let pipeline = naive_bayes::Pipeline::new();
+ Self { pipeline }
+ }
+
+ pub fn fit(&mut self, datapoints: &[Datapoint]) {
+ let datapoints: Vec<_> = datapoints
+ .iter()
+ .map(|datapoint| (normalize(&datapoint.text), datapoint.label))
+ .collect();
+ self.pipeline.fit(&datapoints);
+ }
+
+ pub fn predict_text(&self, text: &str) -> naive_bayes::Prediction<Label> {
+ let text = normalize(text);
+ self.pipeline.predict(&text)
+ }
+
+ pub fn predict(&self, page: &crate::webpage::Webpage) -> naive_bayes::Prediction<Label> {
+ let text = normalize(&page_text(page));
+ self.predict_text(&text)
+ }
+
+ pub fn evaluate(&self, datapoints: &[Datapoint]) -> Evaluation {
+ let mut true_positives = 0;
+ let mut false_positives = 0;
+ let mut true_negatives = 0;
+ let mut false_negatives = 0;
+
+ for datapoint in datapoints {
+ let pred = self.predict_text(&datapoint.text);
+
+ match (pred.label, datapoint.label) {
+ (Label::NSFW, Label::NSFW) => true_positives += 1,
+ (Label::NSFW, Label::SFW) => false_positives += 1,
+ (Label::SFW, Label::SFW) => true_negatives += 1,
+ (Label::SFW, Label::NSFW) => false_negatives += 1,
+ }
+
+ if pred.label != datapoint.label {
+ tracing::debug!(
+ "got {:?} expected {:?} ({:.2}):",
+ pred.label,
+ datapoint.label,
+ pred.confidence
+ );
+ tracing::debug!("{}\n", datapoint.text);
+ }
+ }
+
+ let accuracy = (true_positives + true_negatives) as f64 / datapoints.len() as f64;
+ let precision = true_positives as f64 / (true_positives + false_positives) as f64;
+ let recall = true_positives as f64 / (true_positives + false_negatives) as f64;
+ let f1 = 2.0 * (precision * recall) / (precision + recall);
+
+ Evaluation {
+ accuracy,
+ precision,
+ recall,
+ f1,
+ }
+ }
+
+ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
+ let file = OpenOptions::new()
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(path)?;
+
+ bincode::serialize_into(file, &self)?;
+
+ Ok(())
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let file = OpenOptions::new().read(true).open(path)?;
+
+ let model = bincode::deserialize_from(file)?;
+
+ Ok(model)
+ }
+}
diff --git a/frontend/public/js/search_preferences.js b/frontend/public/js/search_preferences.js
new file mode 100644
index 00000000..4c05500a
--- /dev/null
+++ b/frontend/public/js/search_preferences.js
@@ -0,0 +1,18 @@
+const safeSearchOn = document.getElementById('safe-search-on');
+const safeSearchOff = document.getElementById('safe-search-off');
+
+let safeSearch = window.localStorage.getItem('safeSearch') === 'true';
+
+safeSearchOn.checked = safeSearch;
+safeSearchOff.checked = !safeSearch;
+
+safeSearchOn.addEventListener('change', () => {
+ safeSearch = true;
+ window.localStorage.setItem('safeSearch', safeSearch);
+});
+
+safeSearchOff.addEventListener('change', () => {
+ safeSearch = false;
+ window.localStorage.setItem('safeSearch', safeSearch);
+});
+
diff --git a/frontend/src/components/Searchbar.astro b/frontend/src/components/Searchbar.astro
index 5e325e32..a384c57f 100644
--- a/frontend/src/components/Searchbar.astro
+++ b/frontend/src/components/Searchbar.astro
@@ -67,6 +67,7 @@ const { autofocus, query } = Astro.props;
</div>
<input type="hidden" value="" name="sr" id="siteRankingsUuid" />
+ <input type="hidden" value="" name="ss" id="safeSearch" />
</form>
<script is:inline src="/js/lz-string.min.js"></script>
@@ -76,6 +77,8 @@ const { autofocus, query } = Astro.props;
document.querySelector("#searchbar-form")!;
const siteRankingsUuid: HTMLInputElement =
document.querySelector("#siteRankingsUuid")!;
+ const safeSearch: HTMLInputElement =
+ document.querySelector("#safeSearch")!;
const searchbarInput: HTMLInputElement =
document.querySelector(".searchbar-input")!;
@@ -265,7 +268,12 @@ const { autofocus, query } = Astro.props;
this._submit();
};
- searchbarForm.addEventListener("submit", () => encodeSiteRankings(), false);
+ searchbarForm.addEventListener("submit", () => encodeClientData(), false);
+
+ function encodeClientData() {
+ encodeSiteRankings();
+ encodeSafeSearch();
+ }
function encodeSiteRankings() {
const BLOCKED = 0;
@@ -299,4 +307,9 @@ const { autofocus, query } = Astro.props;
siteRankingsUuid.value = compressed;
}
+
+ function encodeSafeSearch() {
+ let safeSearchValue = window.localStorage.getItem("safeSearch") || "false";
+ safeSearch.value = safeSearchValue;
+ }
</script>
diff --git a/frontend/src/components/SettingsMenu.astro b/frontend/src/components/SettingsMenu.astro
index 570f448d..103cca70 100644
--- a/frontend/src/components/SettingsMenu.astro
+++ b/frontend/src/components/SettingsMenu.astro
@@ -1,6 +1,7 @@
---
const links = [
- { url: "/settings", title: "Manage Optics" },
+ { url: "/settings", title: "Preferences" },
+ { url: "/settings/optics", title: "Manage Optics" },
{ url: "/settings/sites", title: "Site Rankings" },
{ url: "/settings/privacy", title: "Privacy" },
];
diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdx
index e8f0a2b7..ee10c44f 100644
--- a/frontend/src/pages/crawler.mdx
+++ b/frontend/src/pages/crawler.mdx
@@ -10,7 +10,7 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open so
## Politeness
StractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.
-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 4 sec), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 4 sec between requests and at most 60 seconds.
+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 1 sec), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 1 sec between requests and at most 60 seconds.
This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).
The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.
diff --git a/frontend/src/pages/index.astro b/frontend/src/pages/index.astro
index 054e54aa..f11d664a 100644
--- a/frontend/src/pages/index.astro
+++ b/frontend/src/pages/index.astro
@@ -19,7 +19,7 @@ import OpticSelector from "../components/OpticSelector.astro";
</div>
<Searchbar autofocus />
<div class="mt-3 text-gray-600 flex space-x-2">
- <div>Customise your search with an <a class="underline font-medium" href="/settings">optic</a>:</div><OpticSelector default_optics="default_optics" search_on_change={false}/>
+ <div>Customise your search with an <a class="underline font-medium" href="/settings/optics">optic</a>:</div><OpticSelector default_optics="default_optics" search_on_change={false}/>
</div>
</div>
</div>
diff --git a/frontend/src/pages/settings/index.astro b/frontend/src/pages/settings/index.astro
index aaadf434..0770ce73 100644
--- a/frontend/src/pages/settings/index.astro
+++ b/frontend/src/pages/settings/index.astro
@@ -1,64 +1,29 @@
---
import SettingsLayout from "../../layouts/Settings.astro";
-import { askama } from "../../askama";
---
<SettingsLayout>
<div class="content">
<div class="space-y-3">
- <h1 class="text-2xl font-medium">Manage Optics</h1>
- <div class="text-sm">
- Optics lets you control almost everything about which search results
- that gets returned to you. You can discard results from specific
- sites, boost results from other sites and much more.
- </div>
- <div class="text-sm">
- See our <a
- href="https://github.com/StractOrg/sample-optics/blob/main/quickstart.optic"
- >quickstart</a
- > for how to get started. Once you have developed your optic, you can add
- it here to be used during your search.
- </div>
- <div class="text-sm">
- Simply host the optic on a url that returns a plain-text HTTP response
- with the optic. We use raw.githubusercontent.com, but you are free to
- host them elsewhere.
- </div>
- </div>
- <div
- class="mt-16"
- >
- <div class="flex w-full justify-between px-5 h-10">
- <input id="name" type="text" placeholder="Name" />
- <input id="url" type="text" placeholder="Url" />
- <button
- class="w-20 rounded-full border-0 bg-brand text-sm text-white"
- id="add-btn"
- >Add</button
- >
- </div>
- <div class="flex w-full px-5 h-10 mt-2">
- <input id="description" type="text" placeholder="Description" />
- </div>
- <div class="mt-5">
- <div class="grid w-full space-y-5" id="optics-list">
- <div class="flex pl-10">
- <div class="flex-1 font-medium mr-5">Name</div>
- <div class="flex-1 font-medium mr-5">Description</div>
- <div class="flex-1 font-medium">Link</div>
- </div>
- {
- askama.for_("optic in default_optics", () => (
- <div class="flex">
- <div class="mr-5 w-5">
- <img src="/images/disabled-delete.svg" class="h-5 w-5" />
- </div>
- <div class="col-data mr-5">{askama`optic.name $ {{random.words}}`}</div>
- <div class="col-data mr-5">{askama`optic.description $ {{random.words}}`}</div>
- <div class="col-data"><a href={askama`optic.url $ {{internet.url}}`}>Source</a></div>
+ <h1 class="text-2xl font-medium">Preferences</h1>
+ <div>
+ <div>
+ <h2 class="text-lg">Safe Search</h2>
+ <div class="flex flex-col sm:flex-row sm:justify-between pr-5 space-y-1 sm:space-y-0">
+ <p>Remove explicit content from search results</p>
+ <div class="flex space-x-4">
+ <div>
+ <input type="radio" name="safe-search" id="safe-search-on" value="on">
+ <label for="safe-search-on">On</label>
+ </div>
+ <div>
+ <input type="radio" name="safe-search" id="safe-search-off" value="off" checked>
+ <label for="safe-search-off">Off</label>
</div>
- ))
- }
+ </div>
+ </div>
+
+ <p class="pt-5 text-neutral-300">More to come soon</p>
</div>
</div>
</div>
@@ -70,21 +35,6 @@ import { askama } from "../../askama";
text-decoration: underline;
font-weight: 500;
}
-
- .delete-btn {
- width: 1.25rem;
- height: 1.25rem;
- }
-
- .delete-btn:hover {
- cursor: pointer;
- }
-
- .col-data {
- flex: 1 1 0%;
- font-size: 0.875rem;
- line-height: 1.25rem;
- }
</style>
-<script is:inline src="/js/optics_settings.js"></script>
\ No newline at end of file
+<script is:inline src="/js/search_preferences.js"></script>
\ No newline at end of file
diff --git a/frontend/src/pages/settings/optics.astro b/frontend/src/pages/settings/optics.astro
new file mode 100644
index 00000000..aaadf434
--- /dev/null
+++ b/frontend/src/pages/settings/optics.astro
@@ -0,0 +1,90 @@
+---
+import SettingsLayout from "../../layouts/Settings.astro";
+import { askama } from "../../askama";
+---
+
+<SettingsLayout>
+ <div class="content">
+ <div class="space-y-3">
+ <h1 class="text-2xl font-medium">Manage Optics</h1>
+ <div class="text-sm">
+ Optics lets you control almost everything about which search results
+ that gets returned to you. You can discard results from specific
+ sites, boost results from other sites and much more.
+ </div>
+ <div class="text-sm">
+ See our <a
+ href="https://github.com/StractOrg/sample-optics/blob/main/quickstart.optic"
+ >quickstart</a
+ > for how to get started. Once you have developed your optic, you can add
+ it here to be used during your search.
+ </div>
+ <div class="text-sm">
+ Simply host the optic on a url that returns a plain-text HTTP response
+ with the optic. We use raw.githubusercontent.com, but you are free to
+ host them elsewhere.
+ </div>
+ </div>
+ <div
+ class="mt-16"
+ >
+ <div class="flex w-full justify-between px-5 h-10">
+ <input id="name" type="text" placeholder="Name" />
+ <input id="url" type="text" placeholder="Url" />
+ <button
+ class="w-20 rounded-full border-0 bg-brand text-sm text-white"
+ id="add-btn"
+ >Add</button
+ >
+ </div>
+ <div class="flex w-full px-5 h-10 mt-2">
+ <input id="description" type="text" placeholder="Description" />
+ </div>
+ <div class="mt-5">
+ <div class="grid w-full space-y-5" id="optics-list">
+ <div class="flex pl-10">
+ <div class="flex-1 font-medium mr-5">Name</div>
+ <div class="flex-1 font-medium mr-5">Description</div>
+ <div class="flex-1 font-medium">Link</div>
+ </div>
+ {
+ askama.for_("optic in default_optics", () => (
+ <div class="flex">
+ <div class="mr-5 w-5">
+ <img src="/images/disabled-delete.svg" class="h-5 w-5" />
+ </div>
+ <div class="col-data mr-5">{askama`optic.name $ {{random.words}}`}</div>
+ <div class="col-data mr-5">{askama`optic.description $ {{random.words}}`}</div>
+ <div class="col-data"><a href={askama`optic.url $ {{internet.url}}`}>Source</a></div>
+ </div>
+ ))
+ }
+ </div>
+ </div>
+ </div>
+ </div>
+</SettingsLayout>
+
+<style is:inline>
+ .content a {
+ text-decoration: underline;
+ font-weight: 500;
+ }
+
+ .delete-btn {
+ width: 1.25rem;
+ height: 1.25rem;
+ }
+
+ .delete-btn:hover {
+ cursor: pointer;
+ }
+
+ .col-data {
+ flex: 1 1 0%;
+ font-size: 0.875rem;
+ line-height: 1.25rem;
+ }
+</style>
+
+<script is:inline src="/js/optics_settings.js"></script>
\ No newline at end of file
diff --git a/optics/testcases b/optics/testcases
index a772c39c..fa07477c 160000
--- a/optics/testcases
+++ b/optics/testcases
@@ -1 +1 @@
-Subproject commit a772c39ce00edf8b3a810437e6db162ded74bd17
+Subproject commit fa07477c24b2fd0dc6022ca2b139be01edcaaedf
diff --git a/scripts/generate_nsfw_dataset.py b/scripts/generate_nsfw_dataset.py
new file mode 100755
index 00000000..580a3fb8
--- /dev/null
+++ b/scripts/generate_nsfw_dataset.py
@@ -0,0 +1,5121 @@
+#!.venv/bin/python3
+import requests
+import pandas as pd
+from tqdm import tqdm
+import time
+
+stopwords = [
+ "the",
+ "i",
+ "we",
+ "our",
+ "he",
+ "she",
+ "they",
+ "it",
+ "a",
+ "is",
+]
+
+queries = set(
+ stopwords
+ + [
+ "season",
+ "call",
+ "aol mail login",
+ "ao3",
+ "france",
+ "vanguard login",
+ "kanye",
+ "recipe",
+ "zion",
+ "garfield",
+ "amazon customer service",
+ "height",
+ "apple stock",
+ "qr code generator",
+ "ups tracking",
+ "uk",
+ "lakers",
+ "aioli",
+ "food",
+ "credit karma",
+ "american express",
+ "airline tickets",
+ "adidas kanye",
+ "kanye",
+ "airbnb",
+ "amazon",
+ "python",
+ "ant man",
+ "aol mail",
+ "apple stock",
+ "apple",
+ "ariana grande",
+ "papa",
+ ]
+)
+
+
+## exported from https://trystract.com/explore
+nsfw_optic = """DiscardNonMatching;
+Rule {
+ Matches {
+ Site("|youporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|redtube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornhub.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|youjizz.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornone.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|eporner.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|4tube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|spankbang.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tube8.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xhamster.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xnxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porntrex.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xvideos.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|beeg.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porntube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|alohatube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porndig.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cliphunter.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gotporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hqporner.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pichunter.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gelbooru.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|luscious.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|thumbzilla.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porndoe.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|literotica.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornhd.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ixxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|imagefap.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|watchmygf.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tnaflix.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|simply-hentai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lobstertube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubegalore.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornpics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|drtuber.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornmd.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pictoa.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sxyprn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cartoonpornvideos.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nudevista.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ok.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|rule34.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentai2read.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|maturetube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porn300.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|3movs.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nhentai.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|findtubes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dinotube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaifox.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yuvutu.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|myhentaicomics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornsos.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fuskator.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ro89.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|camwhores.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mylust.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sex.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|melonstube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hanime.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zzcartoon.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hclips.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|thisvid.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|voyeurhit.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porndex.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|smutty.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hoodamateurs.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vipergirls.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porndish.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xmoviesforyou.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bellesa.co|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|shesfreaky.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|trannytube.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|multporn.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|8muses.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hotscope.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornbb.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|erosberry.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|txxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|analdin.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|megatube.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xanimeporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nude-gals.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vjav.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ashemaletube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tiava.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fuqer.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lushstories.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nifty.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|asmhentai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fuq.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubesafari.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|anysex.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fux.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|plusone8.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaigasm.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|freeadultcomix.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|animeidhentai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornstarbyface.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|miohentai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|allporncomic.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|muchohentai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dirtyship.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|naughtymachinima.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|camwhoresbay.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|shemalez.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sextvx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|boundhub.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|assoass.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|celebjihad.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|planetsuzy.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hqbabes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lesbianpornvideos.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xfantazy.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornhat.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cumlouder.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubepornstars.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|perfectgirls.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zoig.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porzo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|shooshtime.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|e-hentai.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fullporner.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kindgirls.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|recurbate.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hdzog.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pmatehunter.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|forhertube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|rexxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentai-moon.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaipros.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|3xplanet.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaidude.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vintage-erotica-forum.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexcelebrity.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexlikereal.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|flirt4free.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|trendyporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zbporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|24porn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fapmeifyoucan.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornky.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xpee.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|colegialasdeverdad.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornpaw.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|supjav.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|91porn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|reallifecam.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xmegadrive.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ohentai.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|avgle.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|videobox.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexvid.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|underhentai.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hitomi.la|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|celebsroulette.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jav.guru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|doujins.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|erome.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mypornstarbook.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|aznude.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porndune.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|shameless.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yespornplease.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|svscomics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|punishbang.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zzztube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|namethatporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|storiesonline.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornhits.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xxxtik.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|erofus.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornobae.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|deepfakeporn.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|feet9.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ebonypulse.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cambro.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jjgirls.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|thehentaiworld.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|crazyshit.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|duckgay.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|newestxxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|watchjavonline.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xyzcomics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|javbangers.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|arabysexy.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|4porn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|badjojo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|camshowdownload.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaifromhell.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|shegotass.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexalarab.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bigfuck.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornburst.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porn00.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornerbros.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|anyporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ghettotube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adultdvdtalk.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|noodlemagazine.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cam4.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xtube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.holed.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tabootube.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tastyblacks.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lolhentai.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|faphouse.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|voyeurweb.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xtapes.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porcore.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adultsearch.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|naughtyblog.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaicloud.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|daftsex.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|milffox.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|homemoviestube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|youramateurporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexstories.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|21sextury.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|keezmovies.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|youngpornvideos.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blacked.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bigporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|heavy-r.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexygirlspics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fyptt.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mrdeepfakes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|perfectgirls.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|naoconto.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|myhentaigallery.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vrsmash.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mobifcuk.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|seaporn.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|inporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubepornclassic.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|landing.seancodynetwork.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kitty-kats.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tik.porn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornorips.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornolab.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tsumino.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xozilla.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|h-flash.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|coedcherry.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|novinhasdozapzap.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|anon-v.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xcafe.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubxporn.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|definebabe.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fun.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zdic.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jerkdude.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|primecurves.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gamcore.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wankzvr.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|boobpedia.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kink.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|indiansexstories2.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|prothots.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexjk.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yourfreeporn.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hornywhores.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|titshits.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornplaybb.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|submityourflicks.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornbox.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porntn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adultbay.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xxxvideos247.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nsfw247.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|myporn.club|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|goodporn.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|watch-my-gf.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaiheroes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cosplayporntube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vrlatina.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|whoreshub.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|influencersgonewild.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|letsjerk.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaipulse.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|thotslife.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|flyflv.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porngo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tiktits.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|similar.porn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornxs.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hpjav.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hdporn92.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cremz.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|femefun.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|rule34video.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|scandalplanet.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hobby.porn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hdporncomics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xkeezmovies.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fsicomics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kamababa.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|flingster.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|www5.javmost.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaihere.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|voyeur-house.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sucksex.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bdsmlibrary.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|thothub.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|veporno.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaiporns.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kissjav.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|landing.realitydudesnetwork.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|smutr.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porncomixonline.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|metaporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porndroids.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.seemygf.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hornysimp.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lovehomeporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porn555.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porn4days.biz|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cfake.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|milfnut.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gayforit.eu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|alotporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|poopeegirls.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubebdsm.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mompornonly.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fapvidhd.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|spicybigtits.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|landing.bromonetwork.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|faapy.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|antarvasnaclips.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fap-nation.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|camvideos.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|oncam.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|internetchicks.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sex4arabxxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|damplips.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|carameltube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lewdzone.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|iyottube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|homegrownfreaks.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|register.loveamateur.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|trannyvideosxxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|javfinder.la|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ruleporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|freesexyindians.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornjam.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ebonygalore.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xxxfree.watch|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pururin.io|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexsaoy.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|landing.sweetheartvideo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|enter.avanal.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xanimu.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|brasiltudoliberado.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|netfapx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubepleasure.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fikfap.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|3arabporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nxt-comics.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornbraze.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cartoonporno.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|freeomovie.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|javtiful.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornbay.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|overwatchporn.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornhd3x.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hotmovs.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|empflix.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|whentai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.anal4k.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|babesnetwork.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|julesjordan.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vrhush.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|free-codecs.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|alt.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|iwank.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|alphaporno.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sunporno.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xlovecam.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cams.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vxxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|inhumanity.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|watchhentai.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornolandia.xxx|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaifreak.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|humoron.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|theclassicporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|curvyerotic.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mybigtitsbabes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xfree.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexu.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porntop.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|siterips.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|babesource.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|freehdinterracialporn.in|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|javgg.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mult34.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|meetinchat.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tube.hentaistream.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|secure.anal-angels.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|eroticscribes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.baberotica.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lifeselector.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ebony8.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|upornia.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hdhole.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|assparade.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porncomix.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|access.trueanal.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bang.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|naughtyamericavr.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xnxxarab.cc|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hustler.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|virtualporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nakedpornpics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|t.aagm.link|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|myhentai.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|avn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornkai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pinkdino.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|expatistan.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dropmefiles.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|coolmath.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gomlab.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ezvid.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sweethome3d.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|brdteengal.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubewolf.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sislovesme.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|milfvr.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|huya.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|atspace.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|onlinefreecourse.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|privacypolicytemplate.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yespornpleasexxx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|slutroulette.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexycandidgirls.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|joylovedolls.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.girlcum.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.bbcpie.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vrporn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornoxo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|babepedia.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|analvids.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|digitalplayground.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|czechvr.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|asstr.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pussyspace.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|f95zone.to|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|0xxx.ws|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xxbrits.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adultdeepfakes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nutaku.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|socialmediagirls.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pb-track.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|senzuri.tube|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ftvmilfs.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.amateursexteens.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adultfilmdatabase.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|forum.adultdvdtalk.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hotgaylist.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adultism.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|efukt.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|topescortbabes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mysexgames.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|porn-w.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jacquieetmicheltv.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentaiplay.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kemono.party|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cheggit.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gamesofdesire.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tgtube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|eroticmonkey.ch|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|forumophilia.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|playporngames.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xnalgas.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|coomeet.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|piratecams.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|skipthegames.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hentai.tv|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.myveryfirsttime.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tokyotosho.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|shemalestube.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|landing.mennetwork.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|boodigo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|thenipslip.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fappenist.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bigboobsalert.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lesbian8.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|extreme-board.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sexyandfunny.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.amateureuro.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|secure.anal-beauty.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|join.mamacitaz.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pics-x.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tubedupe.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fakku.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|javhd.today|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vipwank.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornedup.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pornmz.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tsescorts.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sankakucomplex.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|eccie.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xfollow.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|chyoa.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|justpicsplease.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blazinglink.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|handjobhub.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|redditlist.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|trustedshopotc.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jiliblog.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dzenprinimatel.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraonlineslots.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mail-order-wives.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|shoppingcbd.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yuku.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gome.com.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|magix.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|anglican.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|art-psd.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofradeluxeslot.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tlumiki.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|schreibburo.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ferragamo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|infusionsoft.app|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|eacdn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|over40datingsites.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|eastafricangasoil.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|goodlayers.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|phpcms.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gotomoreinfo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|brinkster.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|websitetestlink.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|metu.edu.tr|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tempsite.ws|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|onlinemillionairedatingsites.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ozessay.com.au|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|aussie-pokies.club|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|y0.pl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|redwap.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|goserver.host|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofradeluxe2.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|makepolo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|virginmoneygiving.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ticksy.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gzgov.gov.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|spielen-bookofra.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|my3w.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoilglobal.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|verycd.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|uuu9.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|topasianbrides.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|japanesemailorderbride.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|assignmenthelponline.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|unja.ac.id|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cscse.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|embedly.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|stratteramed.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|trusterworkonline.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xinhua.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|onesmablog.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|valtrexx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jussieu.fr|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|uny.ac.id|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraonlinespiele.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hotrussiangirls.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|c0.pl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-casino.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hookupguru.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|epower.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|home.ne.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|chello.nl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|moy.su|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofradownload.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|interracialdatingsitesreview.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|snnu.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tadalafil247.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cite4me.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tynt.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blogbus.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|atorvastatinlipitor.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|as.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-tricks.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraspill.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|host.sk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|asianwifes.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|best-russian-women.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|clck.yandex.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|track.adsformarket.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|webnode.fr|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|synthroid20.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|advair1.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraonlinegratis.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|admission-essays.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|datarooms.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tinyblogging.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|officelive.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|papascoffee.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tomoreinformation.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|js.digestcolect.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|essaywriter.ca|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bestpornfinder.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gelocal.it|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|virtualave.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|infodeposit.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|iyiou.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jotform.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-onlinespielen.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|albuterolsale.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tsf-ftp.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|alicemchard.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|compaq.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tradetracker.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|webry.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|asianbrides.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|x10host.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|crsky.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|anonymouse.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|onlinespielebookofra.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ueuo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pandora.be|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cool.ne.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|szfw.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cyclopsinfosys.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|russianbrides.us|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|qhub.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dwcdn.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|voila.fr|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|clomidpill.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-online-play.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zhulong.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ucl.ac.be|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|datinglodge.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|urlperu.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra777.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pons.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|webportal.top|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraplayonline.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|neu.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|news12.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zhcw.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ventolinalb.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|orgfree.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blackentertainments.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|antagroup.mn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ccm.gov.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|osdn.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cecdc.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bestrealdatingsites.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yandex.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fivehealthtips.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nlc.gov.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mee.nu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wikia.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|garv.in|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|besthookupsites.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nodak.edu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gkstk.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|typecho.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|my-free.website|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofracards.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|eu5.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yjtag.yahoo.co.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|websitehome.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|salsalabs.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mybeautybrides.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zcool.com.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|asiandatingreviews.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|camgirls1.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zodiyak.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|miaopai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vxinyou.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pointblog.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tillerrakes.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|asianbrides.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|essaywriterforyou.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-paradise.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lz13.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|suhagra2020.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofrakostenlosspiele.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ultius.ws|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zoloftgen.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blastingnews.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoiladvice.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|asian-singles.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lisinopriltab.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nn.pe|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|casinopokies777.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|flywheelsites.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|efu.com.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|poco.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|essaypro.ws|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vox-cdn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|playbookofra.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hotcamgirls1.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bendibao.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraslot.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|chinagwy.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ataraxgen.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|food.blog|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bieberclub.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cas.cz|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraonlinespiele.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|prz.edu.pl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hotbride.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|odn.ne.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|serving-sys.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|trusterworkshop.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|scene7.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gitbooks.io|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pe.hu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xuite.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|els-cdn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bestasianbrides.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraspielenonline.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|csu.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zhiye.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|plaquenilhydroxychloroquine.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|csair.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adsformarket.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|toponlinedatingservices.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|chsi.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dontstopthismusics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|evolutionwriters.biz|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofrasecret.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cheapestpricesale.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|plantronics.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|paipai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|celebrexcap.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|swipnet.se|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-gratis.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ui.ac.id|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|writtingessays.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wifeo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|by.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|b.yjtag.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tw1.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|retinaotc.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|studa.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|playpokiesfree.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hausarbeit-ghostwriter.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yhd.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hbtv.com.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|privatewriting.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|geovisit.ge|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|btinternet.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ccam.org.ar|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vmall.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoilrank.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blog2learn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cngold.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mail-order-wife.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|chemnet.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraspiele.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofradeluxeslot.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hfut.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vietvoters.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|legitmailorderbride.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sitey.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|memberclicks.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|planet.nl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|activehosted.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraonlinespielen.online|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zjut.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|umm.ac.id|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|umk.pl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|slidesharecdn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|onlinebookofraspielen.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dreamessaywriter.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|beep.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|radiovaticana.va|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-online-tricks.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoilworld.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|public.lu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|customessays.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|vhostgo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|datingstudio.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cipro360.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|diowebhost.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraonlinegame.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yjtag.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|multiscreensite.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraspelen.nl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-online-game.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|forumcrea.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kamagra.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|clara.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|russian-women-dating.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|russianbridesfinder.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|playdadnme.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ukrainakomi.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bizland.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|adult-friend-finder.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bbci.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ukraine-women.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|casino-bonus-free-money.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|undip.ac.id|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|acyclovirzov.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gamersky.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|galegroup.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cofc.edu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|diflucanmed.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ptcgeneration.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yourrussianbride.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tokyo.lg.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoildiscount.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ks.gov|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pandora.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ebc.com.br|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|v.calameo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookoframobile.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|oh100.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dlut.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tretinoinsale.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|meitu.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraspel.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|djpodgy.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|err.ee|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gameforge.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|transip.eu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mail-order-brides-sites.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|on.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sm.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pfu.edu.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|domainprofi.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|to8to.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|teacup.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ccnu.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blogolize.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bizrate.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lazaworx.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|digod.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nenu.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|scut.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|domyhomeworkfor.me|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|scriptalicious.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|healthfully.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|dating-ukrainian-brides.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|stagram.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kennesaw.edu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|buyabrideonline.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|valorus-advertising.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wwitv.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|maldimix.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|tv-tv-lv.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-online-spielen.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|blogcn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|amoxicillinbio.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bestlatinabrides.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wedoyouressays.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|moscow-brides.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fontsly.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-player.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|rankmywriter.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|doxycycline360.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|embassy.gov.au|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hostingwijzer.nl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-topliste.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wd.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jnu.edu.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|albendazoleotc.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|besthookupssites.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|propecialab.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|serving.com.ec|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hospedagemdesites.ws|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ocnk.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|whataboutloans.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|synonym.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jewishdatingsites.biz|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra88.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|antabusedsuf.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|msgfocus.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hookupwebsites.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cheapestnetshop.info|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|npage.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|russianbrideswomen.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|baofeng.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|monequateur.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bestlatinwomen.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|aptoide.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kym-cdn.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ym.edu.tw|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|web-pods.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jstv.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-novoline.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoilmarkets.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hunantv.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|qihoo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|es.tl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|haicuneo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|atwebpages.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hs-sites.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|brightbrides.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|isrefer.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mcu.edu.tw|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|suzhou.gov.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|karelia.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|logi.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|rankingsandreviews.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|writing-online.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lockware.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|agri.gov.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|centerblog.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|puzl.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|charmingbrides.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraspielenonline.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|foreign-brides.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-gratis.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ogtk.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|casino-online-australia.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bitcoin.it|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|metformingluc.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|prohosting.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofragratuit.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|freeslotsnodownload-ca.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofrafree.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wps.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|douyu.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|phorum.pl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|netlog.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|indocinmed.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|googlesource.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fd556.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gcs-web.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoilrank.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|myway.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|naturalwellnesscbdoil.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|jouwweb.nl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|besthookup.reviews|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pcgames.com.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ok365.com.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraohnelimits.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|diegoassandri.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|showartcenter.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|funpic.de|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mtvnservices.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-spiel.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gloriousbride.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bestrussianbrides.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|essaywriter24.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kaywa.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|nthu.edu.tw|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|custom-writing.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bestforeignbride.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofradeluxekostenlosspielen.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|freehookup.reviews|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ne.gov|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|gridhosted.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ugu.pl|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|fsnet.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|myfreepokies.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mumayi.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|userapi.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|payforpapers.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|loverusbrides.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|commnet.edu|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mensfitness.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yinyuetai.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|pchouse.com.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|badcreditloanapproving.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|sina.com.tw|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|paytowritemyessay.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hot-russian-women.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|maps.ie|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|webnode.es|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kamagraxr.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|akademitelkom.ac.id|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|zjaic.gov.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|yar.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|php-editors.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|ceair.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|wellbutrinlab.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|findmailorderbride.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|real-money-casino.club|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|lexapro10.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|timepad.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|xnxxxv.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|hospitalathome.it|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|extreme-dm.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|destinyfernandi.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|competitor.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|o2.co.uk|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bofilm.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|qpic.cn|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|livefilestore.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cbdoildelivery.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofra-online.cc|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bridepartner.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|php-myadmin.ru|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|theplatform.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|kir.jp|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofraspiele.net|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|u-strasbg.fr|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|f2s.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|cudasvc.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|mail-order-bride.org|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|bookofrainfo.com|"),
+ } Action(Boost(0))
+};
+Rule {
+ Matches {
+ Site("|scnu.edu.cn|"),
+ } Action(Boost(0))
+};
+Like(Site("pornhub.com"));
+Like(Site("youporn.com"));
+Like(Site("redtube.com"));
+"""
+
+
+def search(json):
+ json["numResults"] = 50
+ r = requests.post(
+ "https://trystract.com/beta/api/search",
+ json=json,
+ ).json()
+
+ time.sleep(5)
+
+ return r
+
+
+def search_nsfw(q):
+ return search({"query": q, "optic": nsfw_optic})
+
+
+def search_sfw(q):
+ return search({"query": q})
+
+
+def content(search_results):
+ return [
+ {"url": result["url"], "text": result["title"] + " " + result["body"]}
+ for result in search_results["webpages"]
+ ]
+
+
+data = {"query": [], "url": [], "text": [], "nsfw": []}
+
+for query in tqdm(queries):
+ results = content(search_nsfw(query))
+ for result in results:
+ data["query"].append(query)
+ data["url"].append(result["url"])
+ data["text"].append(result["text"])
+ data["nsfw"].append(True)
+
+ results = content(search_sfw(query))
+ for result in results:
+ data["query"].append(query)
+ data["url"].append(result["url"])
+ data["text"].append(result["text"])
+ data["nsfw"].append(False)
+
+
+df = pd.DataFrame(data)
+
+df = df.drop_duplicates(subset=["url"])
+
+df = df.rename(columns={"text": "text", "nsfw": "label"})
+df["label"] = df["label"].map({True: "NSFW", False: "SFW"})
+df = df[["text", "label"]]
+
+df.to_csv("data/nsfw.csv", index=False)
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index f41096d8..b834663d 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -4,3 +4,4 @@ torch>=2.0.0
datasets
gensim
safetensors
+pandas
\ No newline at end of file