Skip to content

web search done right

remove image stuff from indexer. When we want it in the future, it should be added to the crawler

Mikkel Denker0ccfb24b1b35
diff --git a/configs/alice.toml b/configs/alice.toml
index 6bb64070..ae1857d9 100644
--- a/configs/alice.toml
+++ b/configs/alice.toml
@@ -4,7 +4,7 @@ host = "0.0.0.0:3004"
gossip_seed_nodes = ["0.0.0.0:3005", "0.0.0.0:3006"]
alice_path = "data/alice"
summarizer_path = "data/summarizer/dual_encoder/"
-encryption_key = "URyJQTjwUjTq6FSRoZGdbUdTIvqs/QxkPacQio8Lhxc="
+encryption_key = "URyJQTjwUjTq6FSRoZGdbUdTIvqs/QxkPacQio8Lhxc=" # don't use this key for anything important
[accelerator]
layer_fraction = 1.0
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index 2d35f966..9eb2afee 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,6 +1,5 @@
centrality_store_path = "./data/centrality"
limit_warc_files = 10
-download_images = false
minimum_clean_words = 40
# crawl_stability_path = "data/crawl-stability"
diff --git a/configs/indexer/profile.toml b/configs/indexer/profile.toml
index 9d5ca99c..b45b7607 100644
--- a/configs/indexer/profile.toml
+++ b/configs/indexer/profile.toml
@@ -1,6 +1,5 @@
warc_paths_file = "warc.paths"
centrality_store_path = "./data/centrality"
-download_images = false
[warc_source]
type = "Local"
diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rs
index b1a3171e..9cec9839 100644
--- a/core/src/entrypoint/alice.rs
+++ b/core/src/entrypoint/alice.rs
@@ -171,6 +171,7 @@ pub async fn run(config: AliceLocalConfig) -> Result<(), crate::alice::Error> {
let addr: SocketAddr = config.host;
let key = base64::decode(config.encryption_key)?;
+ info!("starting alice");
let alice = Alice::open(
&config.alice_path,
&config.summarizer_path,
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 7becea3c..44197027 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -118,7 +118,6 @@ fn create_inverted_index() -> Result<()> {
folder: ".".to_string(),
names: vec![warc_path.to_str().unwrap().to_string()],
}),
- download_images: false,
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
base_path: out_path_tmp.to_str().unwrap().to_string(),
host_centrality_threshold: None,
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 5b4c2b39..43a91ff6 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -50,7 +50,6 @@ pub enum JobConfig {
#[derive(Debug, Serialize, Deserialize)]
pub struct Job {
pub source_config: JobConfig,
- pub download_images: bool,
pub warc_paths: Vec<String>,
pub base_path: String,
pub host_centrality_threshold: Option<f64>,
@@ -202,7 +201,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
page_centrality,
host_centrality,
fetch_time_ms,
- primary_image: None,
pre_computed_score: 0.0,
node_id,
crawl_stability,
@@ -219,10 +217,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
debug!("{:?}", err);
}
}
- if job.download_images {
- info!("downloading images");
- index.download_pending_images();
- }
}
index.commit().unwrap();
@@ -321,7 +315,6 @@ impl Indexer {
.map(|warc_paths| Job {
source_config: job_config.clone(),
warc_paths: warc_paths.collect_vec(),
- download_images: config.download_images.unwrap_or(true),
host_centrality_threshold: config.host_centrality_threshold,
base_path: config
.index_base_path
@@ -406,7 +399,6 @@ impl Indexer {
.map(|warc_paths| Job {
source_config: job_config.clone(),
warc_paths: warc_paths.collect_vec(),
- download_images: config.download_images.unwrap_or(true),
host_centrality_threshold: config.host_centrality_threshold,
base_path: config
.output_path
diff --git a/core/src/image_downloader.rs b/core/src/image_downloader.rs
index 8530c493..3a941d60 100644
--- a/core/src/image_downloader.rs
+++ b/core/src/image_downloader.rs
@@ -128,7 +128,7 @@ where
.drain()
.map(|job| async move { job.download().await }),
)
- .buffer_unordered(20);
+ .buffer_unordered(5);
let mut inserts_since_flush = 0;
diff --git a/core/src/image_store.rs b/core/src/image_store.rs
index 3c9fba42..f5d1e8a2 100644
--- a/core/src/image_store.rs
+++ b/core/src/image_store.rs
@@ -19,12 +19,8 @@ use crate::Result;
use image::imageops::FilterType;
use image::{DynamicImage, ImageOutputFormat};
use serde::{de, ser::SerializeStruct, Serialize};
-use std::collections::HashSet;
use std::io::{Cursor, Read, Seek};
use std::path::Path;
-use uuid::Uuid;
-
-const FAVICON_SIZE: u32 = 32;
#[derive(PartialEq, Debug, Clone)]
pub struct Image(DynamicImage);
@@ -113,10 +109,6 @@ impl BaseImageStore {
self.store.flush();
}
- fn contains(&self, key: &String) -> bool {
- self.store.get(key).is_some()
- }
-
fn get(&self, key: &String) -> Option<Image> {
self.store.get(key)
}
@@ -147,97 +139,6 @@ impl ImageFilter for ResizeFilter {
}
}
-pub struct FaviconStore(BaseImageStore);
-
-impl FaviconStore {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- let store = BaseImageStore::open_with_filters(
- path,
- vec![Box::new(ResizeFilter {
- width: FAVICON_SIZE,
- height: FAVICON_SIZE,
- })],
- );
-
- Self(store)
- }
-
- pub fn contains(&self, key: &String) -> bool {
- self.0.contains(key)
- }
-}
-
-impl ImageStore<String> for FaviconStore {
- fn insert(&mut self, key: String, image: Image) {
- self.0.insert(key, image)
- }
-
- fn get(&self, key: &String) -> Option<Image> {
- self.0.get(key)
- }
-
- fn merge(&mut self, other: Self) {
- self.0.merge(other.0)
- }
-
- fn flush(&self) {
- self.0.flush()
- }
-}
-
-pub struct PrimaryImageStore {
- store: BaseImageStore,
- promised_uuids: HashSet<Uuid>,
-}
-
-impl PrimaryImageStore {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- let store = BaseImageStore::open_with_filters(
- path,
- vec![Box::new(ResizeFilter {
- width: 200,
- height: 100,
- })],
- );
-
- Self {
- store,
- promised_uuids: HashSet::new(),
- }
- }
-
- pub fn generate_uuid(&mut self) -> Uuid {
- let mut uuid = Uuid::new_v4();
-
- while self.store.contains(&uuid.to_string()) || self.promised_uuids.contains(&uuid) {
- uuid = Uuid::new_v4();
- }
-
- self.promised_uuids.insert(uuid);
-
- uuid
- }
-}
-
-impl ImageStore<Uuid> for PrimaryImageStore {
- fn insert(&mut self, uuid: Uuid, image: Image) {
- self.promised_uuids.remove(&uuid);
- self.store.insert(uuid.to_string(), image);
- }
-
- fn get(&self, uuid: &Uuid) -> Option<Image> {
- self.store.get(&uuid.to_string())
- }
-
- fn merge(&mut self, other: Self) {
- self.store.merge(other.store)
- }
-
- fn flush(&self) {
- self.store.flush()
- }
-}
-
pub struct EntityImageStore {
store: BaseImageStore,
}
@@ -330,10 +231,8 @@ mod tests {
let key = "test".to_string();
let mut store = BaseImageStore::open(crate::gen_temp_path());
- assert!(!store.contains(&key));
assert_eq!(store.get(&key), None);
store.insert(key.clone(), image.clone());
- assert!(store.contains(&key));
assert_eq!(store.get(&key), Some(image));
}
@@ -355,31 +254,4 @@ mod tests {
assert_eq!(transformed_image.0.width(), 16);
assert_eq!(transformed_image.0.height(), 16);
}
-
- #[test]
- fn favicon_store() {
- let image = Image(
- ImageBuffer::from_pixel(
- FAVICON_SIZE * 2,
- FAVICON_SIZE * 2,
- image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]),
- )
- .into(),
- );
- assert_eq!(image.0.width(), FAVICON_SIZE * 2);
- assert_eq!(image.0.height(), FAVICON_SIZE * 2);
-
- let mut store = FaviconStore::open(crate::gen_temp_path());
-
- let key = "test".to_string();
-
- assert!(!store.contains(&key));
- assert_eq!(store.get(&key), None);
- store.insert(key.clone(), image);
- assert!(store.contains(&key));
-
- let retrieved_image = store.get(&key).unwrap();
- assert_eq!(retrieved_image.0.width(), FAVICON_SIZE);
- assert_eq!(retrieved_image.0.height(), FAVICON_SIZE);
- }
}
diff --git a/core/src/index.rs b/core/src/index.rs
index 98fb2d25..35fe1cda 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -14,44 +14,32 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::cmp::Ordering;
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::sync::Arc;
-use std::time::Duration;
use serde::{Deserialize, Serialize};
use tantivy::collector::Collector;
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
-use uuid::Uuid;
use crate::directory::{self, DirEntry};
-use crate::image_downloader::{ImageDownloadJob, ImageDownloader};
-use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};
use crate::inverted_index::{self, InvertedIndex};
use crate::query::Query;
use crate::search_ctx::Ctx;
use crate::subdomain_count::SubdomainCounter;
use crate::webgraph::NodeID;
use crate::webpage::region::{Region, RegionCount};
-use crate::webpage::{Url, Webpage};
+use crate::webpage::Webpage;
use crate::Result;
const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";
-const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";
-const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";
const REGION_COUNT_FILE_NAME: &str = "region_count.json";
const SUBDOMAIN_COUNT_SUBFOLDER_NAME: &str = "subdomain_count";
-const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;
pub struct Index {
pub inverted_index: InvertedIndex,
- favicon_store: FaviconStore,
- primary_image_store: PrimaryImageStore,
- favicon_downloader: ImageDownloader<String>,
- primary_image_downloader: ImageDownloader<Uuid>,
pub region_count: RegionCount,
pub subdomain_counter: SubdomainCounter,
pub path: String,
@@ -63,10 +51,6 @@ impl Index {
fs::create_dir_all(path.as_ref())?;
}
- let favicon_store = FaviconStore::open(path.as_ref().join(FAVICON_STORE_SUBFOLDER_NAME));
- let primary_image_store =
- PrimaryImageStore::open(path.as_ref().join(PRIMARY_IMAGE_STORE_SUBFOLDER_NAME));
-
let inverted_index =
InvertedIndex::open(path.as_ref().join(INVERTED_INDEX_SUBFOLDER_NAME))?;
@@ -74,11 +58,7 @@ impl Index {
Ok(Self {
inverted_index,
- favicon_store,
- primary_image_store,
region_count,
- primary_image_downloader: ImageDownloader::new(),
- favicon_downloader: ImageDownloader::new(),
subdomain_counter: SubdomainCounter::open(
path.as_ref().join(SUBDOMAIN_COUNT_SUBFOLDER_NAME),
),
@@ -96,9 +76,7 @@ impl Index {
Self::open(path)
}
- pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {
- self.maybe_insert_favicon(&webpage);
- self.maybe_insert_primary_image(&mut webpage);
+ pub fn insert(&mut self, webpage: Webpage) -> Result<()> {
self.subdomain_counter.increment(webpage.html.url().clone());
if let Ok(region) = Region::guess_from(&webpage) {
@@ -138,30 +116,12 @@ impl Index {
websites: &[inverted_index::WebsitePointer],
query: &Query,
) -> Result<Vec<inverted_index::RetrievedWebpage>> {
- let mut websites = self.inverted_index.retrieve_websites(websites, query)?;
-
- for website in &mut websites {
- let url = Url::from(website.url.clone());
- website.favicon = self.retrieve_favicon(&url);
- if let Some(uuid) = website.primary_image.as_ref().map(|image| &image.uuid) {
- if self.retrieve_primary_image(uuid).is_none() {
- website.primary_image = None;
- }
- }
- }
-
- Ok(websites)
+ self.inverted_index.retrieve_websites(websites, query)
}
pub fn merge(mut self, other: Self) -> Self {
self.inverted_index.merge(other.inverted_index);
- self.favicon_store.merge(other.favicon_store);
- drop(self.favicon_store);
-
- self.primary_image_store.merge(other.primary_image_store);
- drop(self.primary_image_store);
-
self.region_count.merge(other.region_count);
self.subdomain_counter.merge(other.subdomain_counter);
@@ -170,68 +130,10 @@ impl Index {
Self::open(&self.path).expect("failed to open index")
}
- fn maybe_insert_favicon(&mut self, webpage: &Webpage) {
- if !webpage.html.url().is_homepage()
- || self
- .favicon_store
- .contains(&webpage.html.url().domain().to_string())
- {
- return;
- }
-
- if let Some(favicon) = webpage.html.favicon() {
- if favicon.link.is_valid_uri() {
- self.favicon_downloader.schedule(ImageDownloadJob {
- key: favicon.link.domain().to_string(),
- urls: vec![favicon.link],
- timeout: Some(Duration::from_secs(1)),
- });
- }
- }
- }
-
- fn maybe_insert_primary_image(&mut self, webpage: &mut Webpage) {
- match webpage
- .host_centrality
- .partial_cmp(&IMAGE_WEBPAGE_CENTRALITY_THRESHOLD)
- {
- None | Some(Ordering::Greater) => {}
- _ => return,
- }
-
- if let Some(image) = webpage.html.primary_image() {
- let url = image.url.clone();
- if url.is_valid_uri() {
- let uuid = self.primary_image_store.generate_uuid();
- webpage.set_primary_image(uuid, image);
-
- self.primary_image_downloader.schedule(ImageDownloadJob {
- key: uuid,
- urls: vec![url],
- timeout: Some(Duration::from_secs(5)),
- });
- }
- }
- }
-
- pub fn retrieve_favicon(&self, url: &Url) -> Option<Image> {
- self.favicon_store.get(&url.domain().to_string())
- }
-
- pub fn retrieve_primary_image(&self, uuid: &Uuid) -> Option<Image> {
- self.primary_image_store.get(uuid)
- }
-
pub fn schema(&self) -> Arc<Schema> {
self.inverted_index.schema()
}
- pub(crate) fn download_pending_images(&mut self) {
- self.favicon_downloader.download(&mut self.favicon_store);
- self.primary_image_downloader
- .download(&mut self.primary_image_store);
- }
-
pub fn num_segments(&self) -> usize {
self.inverted_index.num_segments()
}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 764befb8..8bf6a6bb 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -26,7 +26,6 @@ use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
use crate::collector::Hashes;
use crate::fastfield_reader::FastFieldReader;
use crate::human_website_annotations::Topic;
-use crate::image_store::Image;
use crate::query::Query;
use crate::ranking::initial::Score;
use crate::ranking::pipeline::RankingWebsite;
@@ -37,7 +36,7 @@ use crate::snippet;
use crate::tokenizer::{BigramTokenizer, Identity, TrigramTokenizer};
use crate::webgraph::NodeID;
use crate::webpage::region::Region;
-use crate::webpage::{schema_org, StoredPrimaryImage, Url, Webpage};
+use crate::webpage::{schema_org, Url, Webpage};
use crate::Result;
use crate::{schema::create_schema, tokenizer::Tokenizer};
use std::collections::HashSet;
@@ -279,22 +278,6 @@ impl InvertedIndex {
.iter()
.map(|website| self.retrieve_doc(website.address, &tv_searcher))
.filter_map(|res| res.ok())
- .map(|mut doc| {
- if let Some(image) = doc.primary_image.as_ref() {
- if !query.simple_terms().iter().all(|term| {
- image
- .title_terms
- .contains(term.to_ascii_lowercase().as_str())
- || image
- .description_terms
- .contains(term.to_ascii_lowercase().as_str())
- }) {
- doc.primary_image = None;
- }
- }
-
- doc
- })
.collect();
for page in &mut webpages {
@@ -494,8 +477,6 @@ pub struct RetrievedWebpage {
pub dirty_body: String,
pub description: Option<String>,
pub dmoz_description: Option<String>,
- pub favicon: Option<Image>,
- pub primary_image: Option<StoredPrimaryImage>,
pub updated_time: Option<NaiveDateTime>,
pub host_topic: Option<Topic>,
pub schema_org: Vec<schema_org::Item>,
@@ -543,16 +524,6 @@ impl From<Document> for RetrievedWebpage {
.expect("Url field should be text")
.to_string();
}
- Field::Text(TextField::PrimaryImage) => {
- webpage.primary_image = {
- let bytes = value
- .value
- .as_bytes()
- .expect("Primary image field should be bytes");
-
- bincode::deserialize(bytes).unwrap()
- }
- }
Field::Fast(FastField::LastUpdated) => {
webpage.updated_time = {
let timestamp = value.value.as_u64().unwrap() as i64;
@@ -611,7 +582,7 @@ impl From<Document> for RetrievedWebpage {
#[cfg(test)]
mod tests {
- use maplit::{hashmap, hashset};
+ use maplit::hashmap;
use crate::{
ranking::{Ranker, SignalAggregator},
@@ -879,7 +850,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- primary_image: None,
node_id: None,
host_topic: None,
crawl_stability: 0.0,
@@ -1132,83 +1102,6 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
}
- #[test]
- fn only_show_primary_images_when_relevant() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
-
- let mut webpage = Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <meta property="og:image" content="https://example.com/link_to_image.html" />
- <meta property="og:description" content="This is an image for the test website" />
- <meta property="og:title" content="title" />
- <title>Test website</title>
- </head>
- <body>
- {CONTENT}
- </body>
- </html>
- "#
- ),
- "https://www.example.com",
- );
- let uuid = uuid::uuid!("00000000-0000-0000-0000-ffff00000000");
- webpage.set_primary_image(uuid, webpage.html.primary_image().unwrap());
-
- index.insert(webpage).expect("failed to insert webpage");
- index.commit().expect("failed to commit index");
-
- let ctx = index.local_search_ctx();
- let query = Query::parse(
- &ctx,
- &SearchQuery {
- query: "website".to_string(),
- ..Default::default()
- },
- &index,
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- SignalAggregator::new(Some(&query)),
- ctx.fastfield_reader.clone(),
- );
-
- let result =
- search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
-
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
- assert_eq!(
- result.documents[0].primary_image,
- Some(StoredPrimaryImage {
- uuid,
- title_terms: hashset! {"title".to_string()},
- description_terms: hashset! {"this".to_string(), "is".to_string(), "an".to_string(), "image".to_string(), "for".to_string(), "the".to_string(), "test".to_string(), "website".to_string()}
- })
- );
-
- let query = Query::parse(
- &ctx,
- &SearchQuery {
- query: "best website".to_string(),
- ..Default::default()
- },
- &index,
- )
- .expect("Failed to parse query");
-
- let result =
- search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
-
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
- assert_eq!(result.documents[0].primary_image, None);
- }
-
#[test]
fn id_links_removed_during_indexing() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
diff --git a/core/src/lib.rs b/core/src/lib.rs
index df08eb1a..3f133e65 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -87,7 +87,6 @@ pub struct IndexingMasterConfig {
warc_source: WarcSource,
workers: Vec<String>,
batch_size: Option<usize>,
- download_images: Option<bool>,
host_centrality_threshold: Option<f64>,
index_base_path: Option<String>,
minimum_clean_words: Option<usize>,
@@ -102,7 +101,6 @@ pub struct IndexingLocalConfig {
batch_size: Option<usize>,
webgraph_path: Option<String>,
output_path: Option<String>,
- download_images: Option<bool>,
host_centrality_threshold: Option<f64>,
topics_path: Option<String>,
centrality_store_path: String,
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index b6c9b866..bf3bb015 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -266,7 +266,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -293,7 +293,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.01,
page_centrality: 0.0,
- primary_image: None,
+
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
@@ -396,7 +396,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -422,7 +422,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
- primary_image: None,
+
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
@@ -503,7 +503,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -530,7 +530,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
- primary_image: None,
+
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
@@ -560,7 +560,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
- primary_image: None,
+
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
@@ -668,7 +668,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
dmoz_description: None,
node_id: Some(*graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),
@@ -695,7 +695,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
- primary_image: None,
+
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
@@ -725,7 +725,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.0002,
page_centrality: 0.0,
- primary_image: None,
+
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
@@ -805,7 +805,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -847,7 +847,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
- primary_image: None,
+
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
@@ -986,7 +986,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1062,7 +1062,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1093,7 +1093,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1149,7 +1149,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1209,7 +1209,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1372,7 +1372,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1447,7 +1447,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1509,7 +1509,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1540,7 +1540,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
@@ -1640,7 +1640,7 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
node_id: None,
host_topic: None,
dmoz_description: None,
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 74e6482e..51b00ede 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -333,7 +333,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),
dmoz_description: None,
@@ -363,7 +363,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),
dmoz_description: None,
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index d90b9c2e..9147cd38 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -206,7 +206,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -236,7 +236,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -285,7 +285,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -315,7 +315,7 @@ mod tests {
// page_centrality: 5.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -363,7 +363,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -393,7 +393,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -421,7 +421,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -469,7 +469,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -499,7 +499,7 @@ mod tests {
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
// page_centrality: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -543,7 +543,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -582,7 +582,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -631,7 +631,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -658,7 +658,7 @@ mod tests {
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
// page_centrality: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: None,
// dmoz_description: None,
@@ -703,7 +703,7 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
page_centrality: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -731,7 +731,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -760,7 +760,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -835,7 +835,7 @@ mod tests {
crawl_stability: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -869,7 +869,7 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
page_centrality: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -903,7 +903,7 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
page_centrality: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -959,7 +959,7 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
page_centrality: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -989,7 +989,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -1039,7 +1039,7 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 1.0,
page_centrality: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -1069,7 +1069,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
crawl_stability: 0.0,
- primary_image: None,
+
host_topic: None,
node_id: None,
dmoz_description: None,
@@ -1126,7 +1126,7 @@ mod tests {
// pre_computed_score: 0.0,
// crawl_stability: 1.0,
// page_centrality: 0.0,
- // primary_image: None,
+ //
// host_topic: Some(topic_a.clone()),
// node_id: Some(1.into()),
// dmoz_description: None,
@@ -1161,7 +1161,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: Some(topic_b.clone()),
// node_id: Some(2.into()),
// dmoz_description: None,
@@ -1196,7 +1196,7 @@ mod tests {
// page_centrality: 0.0,
// pre_computed_score: 0.0,
// crawl_stability: 0.0,
- // primary_image: None,
+ //
// host_topic: None,
// node_id: Some(0.into()),
// dmoz_description: None,
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index f925e133..0188945a 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -85,7 +85,7 @@ mod tests {
fetch_time_ms: 50,
pre_computed_score: 0.0,
page_centrality: 0.0,
- primary_image: None,
+
crawl_stability: 0.0,
dmoz_description: None,
host_topic: None,
@@ -120,7 +120,7 @@ mod tests {
fetch_time_ms: 49,
page_centrality: 0.0,
pre_computed_score: 0.0,
- primary_image: None,
+
crawl_stability: 0.0,
host_topic: None,
dmoz_description: None,
@@ -155,7 +155,7 @@ mod tests {
fetch_time_ms: 0,
page_centrality: 0.0,
pre_computed_score: 0.0,
- primary_image: None,
+
crawl_stability: 0.0,
dmoz_description: None,
host_topic: None,
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 7e4a6dab..f768280e 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use tantivy::schema::{
- BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
+ Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
};
use crate::tokenizer::{BigramTokenizer, Identity, JsonField, Tokenizer, TrigramTokenizer};
@@ -44,7 +44,6 @@ pub enum TextField {
TitleIfHomepage,
BacklinkText,
Description,
- PrimaryImage,
HostTopic,
DmozDescription,
SchemaOrgJson,
@@ -80,7 +79,6 @@ impl TextField {
TextField::DomainNameIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::TitleIfHomepage => Tokenizer::default(),
TextField::BacklinkText => Tokenizer::default(),
- TextField::PrimaryImage => Tokenizer::Identity(Identity {}),
TextField::Description => Tokenizer::default(),
TextField::HostTopic => Tokenizer::default(),
TextField::DmozDescription => Tokenizer::default(),
@@ -119,7 +117,6 @@ impl TextField {
TextField::DomainNameIfHomepageNoTokenizer => false,
TextField::TitleIfHomepage => false,
TextField::BacklinkText => false,
- TextField::PrimaryImage => false,
TextField::Description => true,
TextField::HostTopic => false,
TextField::DmozDescription => true,
@@ -149,7 +146,6 @@ impl TextField {
TextField::DomainIfHomepage => "domain_if_homepage",
TextField::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",
TextField::Description => "description",
- TextField::PrimaryImage => "primary_image_uuid",
TextField::TitleIfHomepage => "title_if_homepage",
TextField::AllBody => "all_body",
TextField::HostTopic => "host_topic",
@@ -243,7 +239,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 54] = [
+pub static ALL_FIELDS: [Field; 53] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -260,7 +256,6 @@ pub static ALL_FIELDS: [Field; 54] = [
Field::Text(TextField::DomainNameIfHomepageNoTokenizer),
Field::Text(TextField::TitleIfHomepage),
Field::Text(TextField::BacklinkText),
- Field::Text(TextField::PrimaryImage),
Field::Text(TextField::Description),
Field::Text(TextField::HostTopic),
Field::Text(TextField::DmozDescription),
@@ -362,9 +357,6 @@ impl Field {
Field::Text(TextField::StemmedCleanBody) => {
IndexingOption::Text(self.default_text_options().set_stored())
}
- Field::Text(TextField::PrimaryImage) => {
- IndexingOption::Bytes(BytesOptions::default().set_stored())
- }
Field::Text(TextField::Description) => {
IndexingOption::Text(self.default_text_options().set_stored())
}
@@ -528,8 +520,7 @@ impl Field {
pub fn is_searchable(&self) -> bool {
!matches!(
self,
- Field::Text(TextField::PrimaryImage)
- | Field::Text(TextField::BacklinkText)
+ Field::Text(TextField::BacklinkText)
| Field::Text(TextField::HostTopic)
| Field::Text(TextField::SchemaOrgJson)
| Field::Text(TextField::CleanBodyBigrams)
@@ -566,7 +557,6 @@ pub fn create_schema() -> tantivy::schema::Schema {
IndexingOption::Text(options) => builder.add_text_field(field.name(), options),
IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),
- IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),
IndexingOption::Facet(options) => builder.add_facet_field(field.name(), options),
};
}
@@ -578,7 +568,6 @@ pub enum IndexingOption {
Text(tantivy::schema::TextOptions),
Integer(tantivy::schema::NumericOptions),
Float(tantivy::schema::NumericOptions),
- Bytes(tantivy::schema::BytesOptions),
Facet(tantivy::schema::FacetOptions),
}
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index 5b7c8093..b5eb0f56 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -157,12 +157,10 @@ pub struct DisplayedWebpage {
pub title: String,
pub url: String,
pub site: String,
- pub favicon_base64: String,
pub domain: String,
pub pretty_url: String,
pub snippet: Snippet,
pub body: String,
- pub primary_image_uuid: Option<String>,
pub ranking_signals: Option<HashMap<Signal, f64>>,
}
@@ -186,21 +184,14 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
let title = html_escape(&webpage.title);
- let favicon_bytes = webpage
- .favicon
- .map(|favicon| favicon.as_raw_bytes())
- .unwrap_or_else(|| include_bytes!("../../../frontend/dist/images/globe.png").to_vec());
-
Self {
title,
site: url.site().to_string(),
url: webpage.url,
pretty_url,
domain,
- favicon_base64: base64::encode(favicon_bytes),
snippet,
body: webpage.body,
- primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),
ranking_signals: None,
}
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 65d0a75f..5d2393d1 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -15,11 +15,8 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::collections::HashMap;
-use std::str::FromStr;
use std::sync::Arc;
-use uuid::Uuid;
-
use crate::entity_index::{EntityIndex, StoredEntity};
use crate::image_store::Image;
use crate::index::Index;
@@ -39,7 +36,6 @@ use crate::spell::Spell;
use crate::webgraph::centrality::topic::TopicCentrality;
use crate::webgraph::Node;
use crate::webpage::region::Region;
-use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
use super::WebsitesResult;
@@ -380,17 +376,6 @@ impl LocalSearcher {
})
}
- pub fn favicon(&self, site: &Url) -> Option<Image> {
- self.index.retrieve_favicon(site)
- }
-
- pub fn primary_image(&self, uuid: String) -> Option<Image> {
- if let Ok(uuid) = Uuid::from_str(uuid.as_str()) {
- return self.index.retrieve_primary_image(&uuid);
- }
- None
- }
-
pub fn entity_image(&self, entity: String) -> Option<Image> {
self.entity_index
.as_ref()
@@ -449,7 +434,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- primary_image: None,
node_id: None,
host_topic: None,
crawl_stability: 0.0,
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 744afffc..11387f9d 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -25,13 +25,8 @@ use chrono::{DateTime, FixedOffset, Utc};
use itertools::Itertools;
use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
use regex::Regex;
-use serde::{Deserialize, Serialize};
-use std::{
- collections::{HashMap, HashSet},
- panic,
-};
+use std::{collections::HashMap, panic};
use tantivy::tokenizer::{PreTokenizedString, Tokenizer};
-use uuid::Uuid;
use whatlang::Lang;
mod just_text;
@@ -115,13 +110,6 @@ impl<const N: usize> Preprocessor<N> {
}
}
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct StoredPrimaryImage {
- pub uuid: Uuid,
- pub title_terms: HashSet<String>,
- pub description_terms: HashSet<String>,
-}
-
pub struct Webpage {
pub html: Html,
pub backlinks: Vec<Link>,
@@ -129,7 +117,6 @@ pub struct Webpage {
pub page_centrality: f64,
pub fetch_time_ms: u64,
pub pre_computed_score: f64,
- pub primary_image: Option<StoredPrimaryImage>,
pub node_id: Option<NodeID>,
pub crawl_stability: f64,
pub host_topic: Option<Topic>,
@@ -148,7 +135,6 @@ impl Webpage {
page_centrality: 0.0,
fetch_time_ms: 0,
pre_computed_score: 0.0,
- primary_image: None,
node_id: None,
host_topic: None,
crawl_stability: 0.0,
@@ -236,14 +222,6 @@ impl Webpage {
self.pre_computed_score,
);
- let image = bincode::serialize(&self.primary_image).unwrap();
- doc.add_bytes(
- schema
- .get_field(Field::Text(TextField::PrimaryImage).name())
- .expect("Failed to get primary_image field"),
- image,
- );
-
doc.add_u64(
schema
.get_field(Field::Fast(FastField::HostNodeID).name())
@@ -279,32 +257,6 @@ impl Webpage {
Ok(doc)
}
-
- pub(crate) fn set_primary_image(&mut self, uuid: Uuid, image: ImageLink) {
- let mut title_terms = HashSet::new();
-
- if let Some(title) = image.title {
- let mut tokenizer = tokenizer::Normal::default().token_stream(title.as_str());
- while let Some(token) = tokenizer.next() {
- title_terms.insert(token.text.clone());
- }
- }
-
- let mut description_terms = HashSet::new();
-
- if let Some(description) = image.description {
- let mut tokenizer = tokenizer::Normal::default().token_stream(description.as_str());
- while let Some(token) = tokenizer.next() {
- description_terms.insert(token.text.clone());
- }
- }
-
- self.primary_image = Some(StoredPrimaryImage {
- uuid,
- title_terms,
- description_terms,
- });
- }
}
struct Script {
@@ -994,8 +946,7 @@ impl Html {
| Field::Fast(FastField::Region)
| Field::Fast(FastField::HostNodeID)
| Field::Fast(FastField::CrawlStability)
- | Field::Text(TextField::DmozDescription)
- | Field::Text(TextField::PrimaryImage) => {}
+ | Field::Text(TextField::DmozDescription) => {}
}
}
@@ -1880,7 +1831,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- primary_image: None,
node_id: None,
crawl_stability: 0.0,
host_topic: None,
@@ -1916,7 +1866,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- primary_image: None,
node_id: None,
crawl_stability: 0.0,
host_topic: None,