web search done right
remove image stuff from indexer. When we want it in the future, it should be added to the crawler
Mikkel Denker0ccfb24b1b35
diff --git a/configs/alice.toml b/configs/alice.tomlindex 6bb64070..ae1857d9 100644--- a/configs/alice.toml+++ b/configs/alice.toml@@ -4,7 +4,7 @@ host = "0.0.0.0:3004"gossip_seed_nodes = ["0.0.0.0:3005", "0.0.0.0:3006"]alice_path = "data/alice"summarizer_path = "data/summarizer/dual_encoder/"-encryption_key = "URyJQTjwUjTq6FSRoZGdbUdTIvqs/QxkPacQio8Lhxc="+encryption_key = "URyJQTjwUjTq6FSRoZGdbUdTIvqs/QxkPacQio8Lhxc=" # don't use this key for anything important[accelerator]layer_fraction = 1.0diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex 2d35f966..9eb2afee 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,6 +1,5 @@centrality_store_path = "./data/centrality"limit_warc_files = 10-download_images = falseminimum_clean_words = 40# crawl_stability_path = "data/crawl-stability"diff --git a/configs/indexer/profile.toml b/configs/indexer/profile.tomlindex 9d5ca99c..b45b7607 100644--- a/configs/indexer/profile.toml+++ b/configs/indexer/profile.toml@@ -1,6 +1,5 @@warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"-download_images = false[warc_source]type = "Local"diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rsindex b1a3171e..9cec9839 100644--- a/core/src/entrypoint/alice.rs+++ b/core/src/entrypoint/alice.rs@@ -171,6 +171,7 @@ pub async fn run(config: AliceLocalConfig) -> Result<(), crate::alice::Error> {let addr: SocketAddr = config.host;let key = base64::decode(config.encryption_key)?;+ info!("starting alice");let alice = Alice::open(&config.alice_path,&config.summarizer_path,diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 7becea3c..44197027 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -118,7 +118,6 @@ fn create_inverted_index() -> Result<()> {folder: ".".to_string(),names: vec![warc_path.to_str().unwrap().to_string()],}),- download_images: false,warc_paths: vec![warc_path.to_str().unwrap().to_string()],base_path: out_path_tmp.to_str().unwrap().to_string(),host_centrality_threshold: None,diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 5b4c2b39..43a91ff6 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -50,7 +50,6 @@ pub enum JobConfig {#[derive(Debug, Serialize, Deserialize)]pub struct Job {pub source_config: JobConfig,- pub download_images: bool,pub warc_paths: Vec<String>,pub base_path: String,pub host_centrality_threshold: Option<f64>,@@ -202,7 +201,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {page_centrality,host_centrality,fetch_time_ms,- primary_image: None,pre_computed_score: 0.0,node_id,crawl_stability,@@ -219,10 +217,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {debug!("{:?}", err);}}- if job.download_images {- info!("downloading images");- index.download_pending_images();- }}index.commit().unwrap();@@ -321,7 +315,6 @@ impl Indexer {.map(|warc_paths| Job {source_config: job_config.clone(),warc_paths: warc_paths.collect_vec(),- download_images: config.download_images.unwrap_or(true),host_centrality_threshold: config.host_centrality_threshold,base_path: config.index_base_path@@ -406,7 +399,6 @@ impl Indexer {.map(|warc_paths| Job {source_config: job_config.clone(),warc_paths: warc_paths.collect_vec(),- download_images: config.download_images.unwrap_or(true),host_centrality_threshold: config.host_centrality_threshold,base_path: config.output_pathdiff --git a/core/src/image_downloader.rs b/core/src/image_downloader.rsindex 8530c493..3a941d60 100644--- a/core/src/image_downloader.rs+++ b/core/src/image_downloader.rs@@ -128,7 +128,7 @@ where.drain().map(|job| async move { job.download().await }),)- .buffer_unordered(20);+ .buffer_unordered(5);let mut inserts_since_flush = 0;diff --git a/core/src/image_store.rs b/core/src/image_store.rsindex 3c9fba42..f5d1e8a2 100644--- a/core/src/image_store.rs+++ b/core/src/image_store.rs@@ -19,12 +19,8 @@ use crate::Result;use image::imageops::FilterType;use image::{DynamicImage, ImageOutputFormat};use serde::{de, ser::SerializeStruct, Serialize};-use std::collections::HashSet;use std::io::{Cursor, Read, Seek};use std::path::Path;-use uuid::Uuid;--const FAVICON_SIZE: u32 = 32;#[derive(PartialEq, Debug, Clone)]pub struct Image(DynamicImage);@@ -113,10 +109,6 @@ impl BaseImageStore {self.store.flush();}- fn contains(&self, key: &String) -> bool {- self.store.get(key).is_some()- }-fn get(&self, key: &String) -> Option<Image> {self.store.get(key)}@@ -147,97 +139,6 @@ impl ImageFilter for ResizeFilter {}}-pub struct FaviconStore(BaseImageStore);--impl FaviconStore {- pub fn open<P: AsRef<Path>>(path: P) -> Self {- let store = BaseImageStore::open_with_filters(- path,- vec![Box::new(ResizeFilter {- width: FAVICON_SIZE,- height: FAVICON_SIZE,- })],- );-- Self(store)- }-- pub fn contains(&self, key: &String) -> bool {- self.0.contains(key)- }-}--impl ImageStore<String> for FaviconStore {- fn insert(&mut self, key: String, image: Image) {- self.0.insert(key, image)- }-- fn get(&self, key: &String) -> Option<Image> {- self.0.get(key)- }-- fn merge(&mut self, other: Self) {- self.0.merge(other.0)- }-- fn flush(&self) {- self.0.flush()- }-}--pub struct PrimaryImageStore {- store: BaseImageStore,- promised_uuids: HashSet<Uuid>,-}--impl PrimaryImageStore {- pub fn open<P: AsRef<Path>>(path: P) -> Self {- let store = BaseImageStore::open_with_filters(- path,- vec![Box::new(ResizeFilter {- width: 200,- height: 100,- })],- );-- Self {- store,- promised_uuids: HashSet::new(),- }- }-- pub fn generate_uuid(&mut self) -> Uuid {- let mut uuid = Uuid::new_v4();-- while self.store.contains(&uuid.to_string()) || self.promised_uuids.contains(&uuid) {- uuid = Uuid::new_v4();- }-- self.promised_uuids.insert(uuid);-- uuid- }-}--impl ImageStore<Uuid> for PrimaryImageStore {- fn insert(&mut self, uuid: Uuid, image: Image) {- self.promised_uuids.remove(&uuid);- self.store.insert(uuid.to_string(), image);- }-- fn get(&self, uuid: &Uuid) -> Option<Image> {- self.store.get(&uuid.to_string())- }-- fn merge(&mut self, other: Self) {- self.store.merge(other.store)- }-- fn flush(&self) {- self.store.flush()- }-}-pub struct EntityImageStore {store: BaseImageStore,}@@ -330,10 +231,8 @@ mod tests {let key = "test".to_string();let mut store = BaseImageStore::open(crate::gen_temp_path());- assert!(!store.contains(&key));assert_eq!(store.get(&key), None);store.insert(key.clone(), image.clone());- assert!(store.contains(&key));assert_eq!(store.get(&key), Some(image));}@@ -355,31 +254,4 @@ mod tests {assert_eq!(transformed_image.0.width(), 16);assert_eq!(transformed_image.0.height(), 16);}-- #[test]- fn favicon_store() {- let image = Image(- ImageBuffer::from_pixel(- FAVICON_SIZE * 2,- FAVICON_SIZE * 2,- image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]),- )- .into(),- );- assert_eq!(image.0.width(), FAVICON_SIZE * 2);- assert_eq!(image.0.height(), FAVICON_SIZE * 2);-- let mut store = FaviconStore::open(crate::gen_temp_path());-- let key = "test".to_string();-- assert!(!store.contains(&key));- assert_eq!(store.get(&key), None);- store.insert(key.clone(), image);- assert!(store.contains(&key));-- let retrieved_image = store.get(&key).unwrap();- assert_eq!(retrieved_image.0.width(), FAVICON_SIZE);- assert_eq!(retrieved_image.0.height(), FAVICON_SIZE);- }}diff --git a/core/src/index.rs b/core/src/index.rsindex 98fb2d25..35fe1cda 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -14,44 +14,32 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::cmp::Ordering;use std::collections::HashSet;use std::fs;use std::path::Path;use std::sync::Arc;-use std::time::Duration;use serde::{Deserialize, Serialize};use tantivy::collector::Collector;use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;-use uuid::Uuid;use crate::directory::{self, DirEntry};-use crate::image_downloader::{ImageDownloadJob, ImageDownloader};-use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};use crate::inverted_index::{self, InvertedIndex};use crate::query::Query;use crate::search_ctx::Ctx;use crate::subdomain_count::SubdomainCounter;use crate::webgraph::NodeID;use crate::webpage::region::{Region, RegionCount};-use crate::webpage::{Url, Webpage};+use crate::webpage::Webpage;use crate::Result;const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";-const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";-const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";const REGION_COUNT_FILE_NAME: &str = "region_count.json";const SUBDOMAIN_COUNT_SUBFOLDER_NAME: &str = "subdomain_count";-const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;pub struct Index {pub inverted_index: InvertedIndex,- favicon_store: FaviconStore,- primary_image_store: PrimaryImageStore,- favicon_downloader: ImageDownloader<String>,- primary_image_downloader: ImageDownloader<Uuid>,pub region_count: RegionCount,pub subdomain_counter: SubdomainCounter,pub path: String,@@ -63,10 +51,6 @@ impl Index {fs::create_dir_all(path.as_ref())?;}- let favicon_store = FaviconStore::open(path.as_ref().join(FAVICON_STORE_SUBFOLDER_NAME));- let primary_image_store =- PrimaryImageStore::open(path.as_ref().join(PRIMARY_IMAGE_STORE_SUBFOLDER_NAME));-let inverted_index =InvertedIndex::open(path.as_ref().join(INVERTED_INDEX_SUBFOLDER_NAME))?;@@ -74,11 +58,7 @@ impl Index {Ok(Self {inverted_index,- favicon_store,- primary_image_store,region_count,- primary_image_downloader: ImageDownloader::new(),- favicon_downloader: ImageDownloader::new(),subdomain_counter: SubdomainCounter::open(path.as_ref().join(SUBDOMAIN_COUNT_SUBFOLDER_NAME),),@@ -96,9 +76,7 @@ impl Index {Self::open(path)}- pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {- self.maybe_insert_favicon(&webpage);- self.maybe_insert_primary_image(&mut webpage);+ pub fn insert(&mut self, webpage: Webpage) -> Result<()> {self.subdomain_counter.increment(webpage.html.url().clone());if let Ok(region) = Region::guess_from(&webpage) {@@ -138,30 +116,12 @@ impl Index {websites: &[inverted_index::WebsitePointer],query: &Query,) -> Result<Vec<inverted_index::RetrievedWebpage>> {- let mut websites = self.inverted_index.retrieve_websites(websites, query)?;-- for website in &mut websites {- let url = Url::from(website.url.clone());- website.favicon = self.retrieve_favicon(&url);- if let Some(uuid) = website.primary_image.as_ref().map(|image| &image.uuid) {- if self.retrieve_primary_image(uuid).is_none() {- website.primary_image = None;- }- }- }-- Ok(websites)+ self.inverted_index.retrieve_websites(websites, query)}pub fn merge(mut self, other: Self) -> Self {self.inverted_index.merge(other.inverted_index);- self.favicon_store.merge(other.favicon_store);- drop(self.favicon_store);-- self.primary_image_store.merge(other.primary_image_store);- drop(self.primary_image_store);-self.region_count.merge(other.region_count);self.subdomain_counter.merge(other.subdomain_counter);@@ -170,68 +130,10 @@ impl Index {Self::open(&self.path).expect("failed to open index")}- fn maybe_insert_favicon(&mut self, webpage: &Webpage) {- if !webpage.html.url().is_homepage()- || self- .favicon_store- .contains(&webpage.html.url().domain().to_string())- {- return;- }-- if let Some(favicon) = webpage.html.favicon() {- if favicon.link.is_valid_uri() {- self.favicon_downloader.schedule(ImageDownloadJob {- key: favicon.link.domain().to_string(),- urls: vec![favicon.link],- timeout: Some(Duration::from_secs(1)),- });- }- }- }-- fn maybe_insert_primary_image(&mut self, webpage: &mut Webpage) {- match webpage- .host_centrality- .partial_cmp(&IMAGE_WEBPAGE_CENTRALITY_THRESHOLD)- {- None | Some(Ordering::Greater) => {}- _ => return,- }-- if let Some(image) = webpage.html.primary_image() {- let url = image.url.clone();- if url.is_valid_uri() {- let uuid = self.primary_image_store.generate_uuid();- webpage.set_primary_image(uuid, image);-- self.primary_image_downloader.schedule(ImageDownloadJob {- key: uuid,- urls: vec![url],- timeout: Some(Duration::from_secs(5)),- });- }- }- }-- pub fn retrieve_favicon(&self, url: &Url) -> Option<Image> {- self.favicon_store.get(&url.domain().to_string())- }-- pub fn retrieve_primary_image(&self, uuid: &Uuid) -> Option<Image> {- self.primary_image_store.get(uuid)- }-pub fn schema(&self) -> Arc<Schema> {self.inverted_index.schema()}- pub(crate) fn download_pending_images(&mut self) {- self.favicon_downloader.download(&mut self.favicon_store);- self.primary_image_downloader- .download(&mut self.primary_image_store);- }-pub fn num_segments(&self) -> usize {self.inverted_index.num_segments()}diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 764befb8..8bf6a6bb 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -26,7 +26,6 @@ use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};use crate::collector::Hashes;use crate::fastfield_reader::FastFieldReader;use crate::human_website_annotations::Topic;-use crate::image_store::Image;use crate::query::Query;use crate::ranking::initial::Score;use crate::ranking::pipeline::RankingWebsite;@@ -37,7 +36,7 @@ use crate::snippet;use crate::tokenizer::{BigramTokenizer, Identity, TrigramTokenizer};use crate::webgraph::NodeID;use crate::webpage::region::Region;-use crate::webpage::{schema_org, StoredPrimaryImage, Url, Webpage};+use crate::webpage::{schema_org, Url, Webpage};use crate::Result;use crate::{schema::create_schema, tokenizer::Tokenizer};use std::collections::HashSet;@@ -279,22 +278,6 @@ impl InvertedIndex {.iter().map(|website| self.retrieve_doc(website.address, &tv_searcher)).filter_map(|res| res.ok())- .map(|mut doc| {- if let Some(image) = doc.primary_image.as_ref() {- if !query.simple_terms().iter().all(|term| {- image- .title_terms- .contains(term.to_ascii_lowercase().as_str())- || image- .description_terms- .contains(term.to_ascii_lowercase().as_str())- }) {- doc.primary_image = None;- }- }-- doc- }).collect();for page in &mut webpages {@@ -494,8 +477,6 @@ pub struct RetrievedWebpage {pub dirty_body: String,pub description: Option<String>,pub dmoz_description: Option<String>,- pub favicon: Option<Image>,- pub primary_image: Option<StoredPrimaryImage>,pub updated_time: Option<NaiveDateTime>,pub host_topic: Option<Topic>,pub schema_org: Vec<schema_org::Item>,@@ -543,16 +524,6 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string();}- Field::Text(TextField::PrimaryImage) => {- webpage.primary_image = {- let bytes = value- .value- .as_bytes()- .expect("Primary image field should be bytes");-- bincode::deserialize(bytes).unwrap()- }- }Field::Fast(FastField::LastUpdated) => {webpage.updated_time = {let timestamp = value.value.as_u64().unwrap() as i64;@@ -611,7 +582,7 @@ impl From<Document> for RetrievedWebpage {#[cfg(test)]mod tests {- use maplit::{hashmap, hashset};+ use maplit::hashmap;use crate::{ranking::{Ranker, SignalAggregator},@@ -879,7 +850,6 @@ mod tests {page_centrality: 0.0,fetch_time_ms: 500,pre_computed_score: 0.0,- primary_image: None,node_id: None,host_topic: None,crawl_stability: 0.0,@@ -1132,83 +1102,6 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com");}- #[test]- fn only_show_primary_images_when_relevant() {- let mut index = InvertedIndex::temporary().expect("Unable to open index");-- let mut webpage = Webpage::new(- &format!(- r#"- <html>- <head>- <meta property="og:image" content="https://example.com/link_to_image.html" />- <meta property="og:description" content="This is an image for the test website" />- <meta property="og:title" content="title" />- <title>Test website</title>- </head>- <body>- {CONTENT}- </body>- </html>- "#- ),- "https://www.example.com",- );- let uuid = uuid::uuid!("00000000-0000-0000-0000-ffff00000000");- webpage.set_primary_image(uuid, webpage.html.primary_image().unwrap());-- index.insert(webpage).expect("failed to insert webpage");- index.commit().expect("failed to commit index");-- let ctx = index.local_search_ctx();- let query = Query::parse(- &ctx,- &SearchQuery {- query: "website".to_string(),- ..Default::default()- },- &index,- )- .expect("Failed to parse query");- let ranker = Ranker::new(- SignalAggregator::new(Some(&query)),- ctx.fastfield_reader.clone(),- );-- let result =- search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");-- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.example.com");- assert_eq!(- result.documents[0].primary_image,- Some(StoredPrimaryImage {- uuid,- title_terms: hashset! {"title".to_string()},- description_terms: hashset! {"this".to_string(), "is".to_string(), "an".to_string(), "image".to_string(), "for".to_string(), "the".to_string(), "test".to_string(), "website".to_string()}- })- );-- let query = Query::parse(- &ctx,- &SearchQuery {- query: "best website".to_string(),- ..Default::default()- },- &index,- )- .expect("Failed to parse query");-- let result =- search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");-- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.example.com");- assert_eq!(result.documents[0].primary_image, None);- }-#[test]fn id_links_removed_during_indexing() {let mut index = InvertedIndex::temporary().expect("Unable to open index");diff --git a/core/src/lib.rs b/core/src/lib.rsindex df08eb1a..3f133e65 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -87,7 +87,6 @@ pub struct IndexingMasterConfig {warc_source: WarcSource,workers: Vec<String>,batch_size: Option<usize>,- download_images: Option<bool>,host_centrality_threshold: Option<f64>,index_base_path: Option<String>,minimum_clean_words: Option<usize>,@@ -102,7 +101,6 @@ pub struct IndexingLocalConfig {batch_size: Option<usize>,webgraph_path: Option<String>,output_path: Option<String>,- download_images: Option<bool>,host_centrality_threshold: Option<f64>,topics_path: Option<String>,centrality_store_path: String,diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex b6c9b866..bf3bb015 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -266,7 +266,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -293,7 +293,7 @@ mod tests {backlinks: vec![],host_centrality: 0.01,page_centrality: 0.0,- primary_image: None,+pre_computed_score: 0.0,crawl_stability: 0.0,fetch_time_ms: 500,@@ -396,7 +396,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -422,7 +422,7 @@ mod tests {backlinks: vec![],host_centrality: 0.0001,page_centrality: 0.0,- primary_image: None,+pre_computed_score: 0.0,crawl_stability: 0.0,fetch_time_ms: 500,@@ -503,7 +503,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -530,7 +530,7 @@ mod tests {backlinks: vec![],host_centrality: 0.0001,page_centrality: 0.0,- primary_image: None,+pre_computed_score: 0.0,crawl_stability: 0.0,fetch_time_ms: 500,@@ -560,7 +560,7 @@ mod tests {backlinks: vec![],host_centrality: 0.0001,page_centrality: 0.0,- primary_image: None,+pre_computed_score: 0.0,crawl_stability: 0.0,fetch_time_ms: 500,@@ -668,7 +668,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,dmoz_description: None,node_id: Some(*graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),@@ -695,7 +695,7 @@ mod tests {backlinks: vec![],host_centrality: 0.0001,page_centrality: 0.0,- primary_image: None,+pre_computed_score: 0.0,crawl_stability: 0.0,fetch_time_ms: 500,@@ -725,7 +725,7 @@ mod tests {backlinks: vec![],host_centrality: 0.0002,page_centrality: 0.0,- primary_image: None,+pre_computed_score: 0.0,crawl_stability: 0.0,fetch_time_ms: 500,@@ -805,7 +805,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -847,7 +847,7 @@ mod tests {backlinks: vec![],host_centrality: 0.0001,page_centrality: 0.0,- primary_image: None,+pre_computed_score: 0.0,crawl_stability: 0.0,fetch_time_ms: 500,@@ -986,7 +986,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1062,7 +1062,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1093,7 +1093,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1149,7 +1149,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1209,7 +1209,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1372,7 +1372,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1447,7 +1447,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1509,7 +1509,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1540,7 +1540,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,@@ -1640,7 +1640,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+node_id: None,host_topic: None,dmoz_description: None,diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 74e6482e..51b00ede 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -333,7 +333,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),dmoz_description: None,@@ -363,7 +363,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),dmoz_description: None,diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex d90b9c2e..9147cd38 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -206,7 +206,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -236,7 +236,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -285,7 +285,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -315,7 +315,7 @@ mod tests {// page_centrality: 5.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -363,7 +363,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -393,7 +393,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -421,7 +421,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -469,7 +469,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -499,7 +499,7 @@ mod tests {// pre_computed_score: 0.0,// crawl_stability: 0.0,// page_centrality: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -543,7 +543,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -582,7 +582,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -631,7 +631,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -658,7 +658,7 @@ mod tests {// pre_computed_score: 0.0,// crawl_stability: 0.0,// page_centrality: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: None,// dmoz_description: None,@@ -703,7 +703,7 @@ mod tests {pre_computed_score: 0.0,crawl_stability: 0.0,page_centrality: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -731,7 +731,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -760,7 +760,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -835,7 +835,7 @@ mod tests {crawl_stability: 0.0,fetch_time_ms: 500,page_centrality: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -869,7 +869,7 @@ mod tests {pre_computed_score: 0.0,crawl_stability: 0.0,page_centrality: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -903,7 +903,7 @@ mod tests {pre_computed_score: 0.0,crawl_stability: 0.0,page_centrality: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -959,7 +959,7 @@ mod tests {pre_computed_score: 0.0,crawl_stability: 0.0,page_centrality: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -989,7 +989,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -1039,7 +1039,7 @@ mod tests {pre_computed_score: 0.0,crawl_stability: 1.0,page_centrality: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -1069,7 +1069,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,crawl_stability: 0.0,- primary_image: None,+host_topic: None,node_id: None,dmoz_description: None,@@ -1126,7 +1126,7 @@ mod tests {// pre_computed_score: 0.0,// crawl_stability: 1.0,// page_centrality: 0.0,- // primary_image: None,+ //// host_topic: Some(topic_a.clone()),// node_id: Some(1.into()),// dmoz_description: None,@@ -1161,7 +1161,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: Some(topic_b.clone()),// node_id: Some(2.into()),// dmoz_description: None,@@ -1196,7 +1196,7 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,// crawl_stability: 0.0,- // primary_image: None,+ //// host_topic: None,// node_id: Some(0.into()),// dmoz_description: None,diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex f925e133..0188945a 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -85,7 +85,7 @@ mod tests {fetch_time_ms: 50,pre_computed_score: 0.0,page_centrality: 0.0,- primary_image: None,+crawl_stability: 0.0,dmoz_description: None,host_topic: None,@@ -120,7 +120,7 @@ mod tests {fetch_time_ms: 49,page_centrality: 0.0,pre_computed_score: 0.0,- primary_image: None,+crawl_stability: 0.0,host_topic: None,dmoz_description: None,@@ -155,7 +155,7 @@ mod tests {fetch_time_ms: 0,page_centrality: 0.0,pre_computed_score: 0.0,- primary_image: None,+crawl_stability: 0.0,dmoz_description: None,host_topic: None,diff --git a/core/src/schema.rs b/core/src/schema.rsindex 7e4a6dab..f768280e 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use tantivy::schema::{- BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,+ Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};use crate::tokenizer::{BigramTokenizer, Identity, JsonField, Tokenizer, TrigramTokenizer};@@ -44,7 +44,6 @@ pub enum TextField {TitleIfHomepage,BacklinkText,Description,- PrimaryImage,HostTopic,DmozDescription,SchemaOrgJson,@@ -80,7 +79,6 @@ impl TextField {TextField::DomainNameIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),TextField::TitleIfHomepage => Tokenizer::default(),TextField::BacklinkText => Tokenizer::default(),- TextField::PrimaryImage => Tokenizer::Identity(Identity {}),TextField::Description => Tokenizer::default(),TextField::HostTopic => Tokenizer::default(),TextField::DmozDescription => Tokenizer::default(),@@ -119,7 +117,6 @@ impl TextField {TextField::DomainNameIfHomepageNoTokenizer => false,TextField::TitleIfHomepage => false,TextField::BacklinkText => false,- TextField::PrimaryImage => false,TextField::Description => true,TextField::HostTopic => false,TextField::DmozDescription => true,@@ -149,7 +146,6 @@ impl TextField {TextField::DomainIfHomepage => "domain_if_homepage",TextField::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",TextField::Description => "description",- TextField::PrimaryImage => "primary_image_uuid",TextField::TitleIfHomepage => "title_if_homepage",TextField::AllBody => "all_body",TextField::HostTopic => "host_topic",@@ -243,7 +239,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 54] = [+pub static ALL_FIELDS: [Field; 53] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -260,7 +256,6 @@ pub static ALL_FIELDS: [Field; 54] = [Field::Text(TextField::DomainNameIfHomepageNoTokenizer),Field::Text(TextField::TitleIfHomepage),Field::Text(TextField::BacklinkText),- Field::Text(TextField::PrimaryImage),Field::Text(TextField::Description),Field::Text(TextField::HostTopic),Field::Text(TextField::DmozDescription),@@ -362,9 +357,6 @@ impl Field {Field::Text(TextField::StemmedCleanBody) => {IndexingOption::Text(self.default_text_options().set_stored())}- Field::Text(TextField::PrimaryImage) => {- IndexingOption::Bytes(BytesOptions::default().set_stored())- }Field::Text(TextField::Description) => {IndexingOption::Text(self.default_text_options().set_stored())}@@ -528,8 +520,7 @@ impl Field {pub fn is_searchable(&self) -> bool {!matches!(self,- Field::Text(TextField::PrimaryImage)- | Field::Text(TextField::BacklinkText)+ Field::Text(TextField::BacklinkText)| Field::Text(TextField::HostTopic)| Field::Text(TextField::SchemaOrgJson)| Field::Text(TextField::CleanBodyBigrams)@@ -566,7 +557,6 @@ pub fn create_schema() -> tantivy::schema::Schema {IndexingOption::Text(options) => builder.add_text_field(field.name(), options),IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),- IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),IndexingOption::Facet(options) => builder.add_facet_field(field.name(), options),};}@@ -578,7 +568,6 @@ pub enum IndexingOption {Text(tantivy::schema::TextOptions),Integer(tantivy::schema::NumericOptions),Float(tantivy::schema::NumericOptions),- Bytes(tantivy::schema::BytesOptions),Facet(tantivy::schema::FacetOptions),}diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsindex 5b7c8093..b5eb0f56 100644--- a/core/src/search_prettifier/mod.rs+++ b/core/src/search_prettifier/mod.rs@@ -157,12 +157,10 @@ pub struct DisplayedWebpage {pub title: String,pub url: String,pub site: String,- pub favicon_base64: String,pub domain: String,pub pretty_url: String,pub snippet: Snippet,pub body: String,- pub primary_image_uuid: Option<String>,pub ranking_signals: Option<HashMap<Signal, f64>>,}@@ -186,21 +184,14 @@ impl From<RetrievedWebpage> for DisplayedWebpage {let title = html_escape(&webpage.title);- let favicon_bytes = webpage- .favicon- .map(|favicon| favicon.as_raw_bytes())- .unwrap_or_else(|| include_bytes!("../../../frontend/dist/images/globe.png").to_vec());-Self {title,site: url.site().to_string(),url: webpage.url,pretty_url,domain,- favicon_base64: base64::encode(favicon_bytes),snippet,body: webpage.body,- primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),ranking_signals: None,}}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 65d0a75f..5d2393d1 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -15,11 +15,8 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::collections::HashMap;-use std::str::FromStr;use std::sync::Arc;-use uuid::Uuid;-use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;@@ -39,7 +36,6 @@ use crate::spell::Spell;use crate::webgraph::centrality::topic::TopicCentrality;use crate::webgraph::Node;use crate::webpage::region::Region;-use crate::webpage::Url;use crate::{inverted_index, Error, Result};use super::WebsitesResult;@@ -380,17 +376,6 @@ impl LocalSearcher {})}- pub fn favicon(&self, site: &Url) -> Option<Image> {- self.index.retrieve_favicon(site)- }-- pub fn primary_image(&self, uuid: String) -> Option<Image> {- if let Ok(uuid) = Uuid::from_str(uuid.as_str()) {- return self.index.retrieve_primary_image(&uuid);- }- None- }-pub fn entity_image(&self, entity: String) -> Option<Image> {self.entity_index.as_ref()@@ -449,7 +434,6 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,- primary_image: None,node_id: None,host_topic: None,crawl_stability: 0.0,diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 744afffc..11387f9d 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -25,13 +25,8 @@ use chrono::{DateTime, FixedOffset, Utc};use itertools::Itertools;use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};use regex::Regex;-use serde::{Deserialize, Serialize};-use std::{- collections::{HashMap, HashSet},- panic,-};+use std::{collections::HashMap, panic};use tantivy::tokenizer::{PreTokenizedString, Tokenizer};-use uuid::Uuid;use whatlang::Lang;mod just_text;@@ -115,13 +110,6 @@ impl<const N: usize> Preprocessor<N> {}}-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]-pub struct StoredPrimaryImage {- pub uuid: Uuid,- pub title_terms: HashSet<String>,- pub description_terms: HashSet<String>,-}-pub struct Webpage {pub html: Html,pub backlinks: Vec<Link>,@@ -129,7 +117,6 @@ pub struct Webpage {pub page_centrality: f64,pub fetch_time_ms: u64,pub pre_computed_score: f64,- pub primary_image: Option<StoredPrimaryImage>,pub node_id: Option<NodeID>,pub crawl_stability: f64,pub host_topic: Option<Topic>,@@ -148,7 +135,6 @@ impl Webpage {page_centrality: 0.0,fetch_time_ms: 0,pre_computed_score: 0.0,- primary_image: None,node_id: None,host_topic: None,crawl_stability: 0.0,@@ -236,14 +222,6 @@ impl Webpage {self.pre_computed_score,);- let image = bincode::serialize(&self.primary_image).unwrap();- doc.add_bytes(- schema- .get_field(Field::Text(TextField::PrimaryImage).name())- .expect("Failed to get primary_image field"),- image,- );-doc.add_u64(schema.get_field(Field::Fast(FastField::HostNodeID).name())@@ -279,32 +257,6 @@ impl Webpage {Ok(doc)}-- pub(crate) fn set_primary_image(&mut self, uuid: Uuid, image: ImageLink) {- let mut title_terms = HashSet::new();-- if let Some(title) = image.title {- let mut tokenizer = tokenizer::Normal::default().token_stream(title.as_str());- while let Some(token) = tokenizer.next() {- title_terms.insert(token.text.clone());- }- }-- let mut description_terms = HashSet::new();-- if let Some(description) = image.description {- let mut tokenizer = tokenizer::Normal::default().token_stream(description.as_str());- while let Some(token) = tokenizer.next() {- description_terms.insert(token.text.clone());- }- }-- self.primary_image = Some(StoredPrimaryImage {- uuid,- title_terms,- description_terms,- });- }}struct Script {@@ -994,8 +946,7 @@ impl Html {| Field::Fast(FastField::Region)| Field::Fast(FastField::HostNodeID)| Field::Fast(FastField::CrawlStability)- | Field::Text(TextField::DmozDescription)- | Field::Text(TextField::PrimaryImage) => {}+ | Field::Text(TextField::DmozDescription) => {}}}@@ -1880,7 +1831,6 @@ mod tests {page_centrality: 0.0,fetch_time_ms: 500,pre_computed_score: 0.0,- primary_image: None,node_id: None,crawl_stability: 0.0,host_topic: None,@@ -1916,7 +1866,6 @@ mod tests {page_centrality: 0.0,fetch_time_ms: 500,pre_computed_score: 0.0,- primary_image: None,node_id: None,crawl_stability: 0.0,host_topic: None,