web search done right
count number of subdomains during indexing
Mikkel Denkera6ea2e7ba0ed
diff --git a/src/index.rs b/src/index.rsindex 85d8aa83..71c35664 100644--- a/src/index.rs+++ b/src/index.rs@@ -33,6 +33,7 @@ use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};use crate::inverted_index::{self, InitialSearchResult, InvertedIndex, SearchResult};use crate::query::Query;use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};+use crate::subdomain_count::SubdomainCounter;use crate::webpage::region::{Region, RegionCount};use crate::webpage::{Url, Webpage};use crate::Result;@@ -42,6 +43,7 @@ const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";const SPELL_SUBFOLDER_NAME: &str = "primary_image_store";const REGION_COUNT_FILE_NAME: &str = "region_count.json";+const SUBDOMAIN_COUNT_SUBFOLDER_NAME: &str = "subdomain_count";const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;pub struct Index {@@ -52,6 +54,7 @@ pub struct Index {primary_image_downloader: ImageDownloader<Uuid>,spell_dictionary: Dictionary<100_000>,pub region_count: RegionCount,+ pub subdomain_counter: SubdomainCounter,pub path: String,}@@ -78,6 +81,9 @@ impl Index {primary_image_downloader: ImageDownloader::new(),favicon_downloader: ImageDownloader::new(),spell_dictionary: Dictionary::open(Some(path.as_ref().join(SPELL_SUBFOLDER_NAME)))?,+ subdomain_counter: SubdomainCounter::open(+ path.as_ref().join(SUBDOMAIN_COUNT_SUBFOLDER_NAME),+ ),path: path.as_ref().to_str().unwrap().to_string(),})}@@ -96,6 +102,7 @@ impl Index {self.maybe_insert_favicon(&webpage);self.maybe_insert_primary_image(&mut webpage);self.spell_dictionary.insert_page(&webpage);+ self.subdomain_counter.increment(webpage.html.url().clone());if let Ok(region) = Region::guess_from(&webpage) {self.region_count.increment(®ion);@@ -108,6 +115,7 @@ impl Index {self.spell_dictionary.commit()?;self.inverted_index.commit()?;self.region_count.commit();+ self.subdomain_counter.commit();Ok(())}diff --git a/src/lib.rs b/src/lib.rsindex 00a11250..8c12a1b6 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -53,6 +53,7 @@ pub mod searcher;mod snippet;mod sonic;mod spell;+mod subdomain_count;mod tokenizer;#[allow(unused)]mod ttl_cache;diff --git a/src/subdomain_count.rs b/src/subdomain_count.rsnew file mode 100644index 00000000..add016f1--- /dev/null+++ b/src/subdomain_count.rs@@ -0,0 +1,49 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ kv::{rocksdb_store::RocksDbStore, Kv},+ prehashed::{hash, Prehashed},+ webpage::Url,+};+use std::{collections::HashSet, path::Path};++pub struct SubdomainCounter {+ inner: Box<dyn Kv<Prehashed, HashSet<String>>>,+}++impl SubdomainCounter {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ Self {+ inner: RocksDbStore::open(path),+ }+ }++ pub fn increment(&mut self, url: Url) {+ if let Some(subdomain) = url.subdomain() {+ let domain = hash(url.domain());+ let subdomain = subdomain.to_string();++ let mut set = self.inner.get(&domain).unwrap_or_default();+ set.insert(subdomain);+ self.inner.insert(domain, set);+ }+ }++ pub fn commit(&self) {+ self.inner.flush();+ }+}