Skip to content

web search done right

count number of subdomains during indexing

Mikkel Denkera6ea2e7ba0ed
diff --git a/src/index.rs b/src/index.rs
index 85d8aa83..71c35664 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -33,6 +33,7 @@ use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};
use crate::inverted_index::{self, InitialSearchResult, InvertedIndex, SearchResult};
use crate::query::Query;
use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};
+use crate::subdomain_count::SubdomainCounter;
use crate::webpage::region::{Region, RegionCount};
use crate::webpage::{Url, Webpage};
use crate::Result;
@@ -42,6 +43,7 @@ const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";
const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";
const SPELL_SUBFOLDER_NAME: &str = "primary_image_store";
const REGION_COUNT_FILE_NAME: &str = "region_count.json";
+const SUBDOMAIN_COUNT_SUBFOLDER_NAME: &str = "subdomain_count";
const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;
pub struct Index {
@@ -52,6 +54,7 @@ pub struct Index {
primary_image_downloader: ImageDownloader<Uuid>,
spell_dictionary: Dictionary<100_000>,
pub region_count: RegionCount,
+ pub subdomain_counter: SubdomainCounter,
pub path: String,
}
@@ -78,6 +81,9 @@ impl Index {
primary_image_downloader: ImageDownloader::new(),
favicon_downloader: ImageDownloader::new(),
spell_dictionary: Dictionary::open(Some(path.as_ref().join(SPELL_SUBFOLDER_NAME)))?,
+ subdomain_counter: SubdomainCounter::open(
+ path.as_ref().join(SUBDOMAIN_COUNT_SUBFOLDER_NAME),
+ ),
path: path.as_ref().to_str().unwrap().to_string(),
})
}
@@ -96,6 +102,7 @@ impl Index {
self.maybe_insert_favicon(&webpage);
self.maybe_insert_primary_image(&mut webpage);
self.spell_dictionary.insert_page(&webpage);
+ self.subdomain_counter.increment(webpage.html.url().clone());
if let Ok(region) = Region::guess_from(&webpage) {
self.region_count.increment(&region);
@@ -108,6 +115,7 @@ impl Index {
self.spell_dictionary.commit()?;
self.inverted_index.commit()?;
self.region_count.commit();
+ self.subdomain_counter.commit();
Ok(())
}
diff --git a/src/lib.rs b/src/lib.rs
index 00a11250..8c12a1b6 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -53,6 +53,7 @@ pub mod searcher;
mod snippet;
mod sonic;
mod spell;
+mod subdomain_count;
mod tokenizer;
#[allow(unused)]
mod ttl_cache;
diff --git a/src/subdomain_count.rs b/src/subdomain_count.rs
new file mode 100644
index 00000000..add016f1
--- /dev/null
+++ b/src/subdomain_count.rs
@@ -0,0 +1,49 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ kv::{rocksdb_store::RocksDbStore, Kv},
+ prehashed::{hash, Prehashed},
+ webpage::Url,
+};
+use std::{collections::HashSet, path::Path};
+
+pub struct SubdomainCounter {
+ inner: Box<dyn Kv<Prehashed, HashSet<String>>>,
+}
+
+impl SubdomainCounter {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ Self {
+ inner: RocksDbStore::open(path),
+ }
+ }
+
+ pub fn increment(&mut self, url: Url) {
+ if let Some(subdomain) = url.subdomain() {
+ let domain = hash(url.domain());
+ let subdomain = subdomain.to_string();
+
+ let mut set = self.inner.get(&domain).unwrap_or_default();
+ set.insert(subdomain);
+ self.inner.insert(domain, set);
+ }
+ }
+
+ pub fn commit(&self) {
+ self.inner.flush();
+ }
+}