web search done right
Unravel recursive functions to avoid stackoverflow. Summarizer is also WIP (there might be some 'never used' warnings when compiling)
Mikkel Denker785e610db8ed
diff --git a/Cargo.lock b/Cargo.lockindex fc0f7067..ba9999e0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -788,6 +788,7 @@ dependencies = ["base64 0.13.1","bincode","bitvec",+ "byteorder","bzip2","chrono","clap 3.2.23",diff --git a/Cargo.toml b/Cargo.tomlindex a8dc6b5a..8ecaf7ee 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -66,4 +66,5 @@ rand = "0.8.5"crossbeam-channel = "0.5.6"num_cpus = "1.15.0"onnxruntime = { git = "https://github.com/Cuely/onnxruntime-rs" }-tokenizers = "0.13.2"\ No newline at end of file+tokenizers = "0.13.2"+byteorder = "1.4.3"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex ba9a28e5..06b1e910 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -72,6 +72,7 @@ num_cpus = { workspace = true }onnxruntime = { workspace = true }tokenizers = { workspace = true }lalrpop-util = { workspace = true }+byteorder = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/core/benches/online-harmonic.rs b/core/benches/online-harmonic.rsindex d9e9c4ac..21c04c36 100644--- a/core/benches/online-harmonic.rs+++ b/core/benches/online-harmonic.rs@@ -15,7 +15,9 @@ pub fn criterion_benchmark(c: &mut Criterion) {let harmonic_centrality_store = HarmonicCentralityStore::open(cuely::gen_temp_path());for (node, centrality) in centrality.host {- harmonic_centrality_store.host.insert(node.name, centrality);+ harmonic_centrality_store+ .host+ .insert(webgraph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 8fe79ea7..496b959d 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -18,12 +18,12 @@ use std::net::SocketAddr;use std::path::Path;use itertools::Itertools;-use rayon::prelude::*;use serde::{Deserialize, Serialize};use tokio::pin;use tracing::{debug, info, trace};use crate::entrypoint::async_download_all_warc_files;+use crate::executor::Executor;use crate::index::{FrozenIndex, Index};use crate::mapreduce::{Manager, Map, Reduce, Worker};use crate::ranking::centrality_store::IndexerCentralityStore;@@ -390,28 +390,30 @@ impl Indexer {config.topics_path.clone(),);- let indexes: Vec<_> = warc_paths- .into_iter()- .skip(config.skip_num_warc_files.unwrap_or(0))- .take(config.limit_warc_files.unwrap_or(usize::MAX))- .chunks(config.batch_size.unwrap_or(1))- .into_iter()- .map(|warc_paths| Job {- source_config: job_config.clone(),- warc_paths: warc_paths.collect_vec(),- download_images: config.download_images.unwrap_or(true),- host_centrality_threshold: config.host_centrality_threshold,- base_path: config- .output_path- .clone()- .unwrap_or_else(|| "data/index".to_string()),- max_num_segments: config.final_num_segments.unwrap_or(20),- })- .collect_vec()- .into_par_iter()- .panic_fuse()- .map(|job| -> IndexPointer { job.map(&worker) })- .collect();+ let executor = Executor::multi_thread("indexer").unwrap();++ let indexes = executor+ .map(+ |job| -> IndexPointer { job.map(&worker) },+ warc_paths+ .into_iter()+ .skip(config.skip_num_warc_files.unwrap_or(0))+ .take(config.limit_warc_files.unwrap_or(usize::MAX))+ .chunks(config.batch_size.unwrap_or(1))+ .into_iter()+ .map(|warc_paths| Job {+ source_config: job_config.clone(),+ warc_paths: warc_paths.collect_vec(),+ download_images: config.download_images.unwrap_or(true),+ host_centrality_threshold: config.host_centrality_threshold,+ base_path: config+ .output_path+ .clone()+ .unwrap_or_else(|| "data/index".to_string()),+ max_num_segments: config.final_num_segments.unwrap_or(20),+ }),+ )+ .unwrap_or_default();Self::merge(indexes, config.final_num_segments.unwrap_or(20))?;Ok(())diff --git a/core/src/webgraph/executor.rs b/core/src/executor.rssimilarity index 99%rename from core/src/webgraph/executor.rsrename to core/src/executor.rsindex 81750750..b3300c88 100644--- a/core/src/webgraph/executor.rs+++ b/core/src/executor.rs@@ -38,7 +38,7 @@ impl Executor {pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {let pool = ThreadPoolBuilder::new().num_threads(num_threads)- .stack_size(8_000_000)+ .stack_size(80_000_000).thread_name(move |num| format!("{prefix}{num}")).build()?;Ok(Executor::ThreadPool(pool))diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rsindex 9ecc8f75..7846fdb3 100644--- a/core/src/hyperloglog.rs+++ b/core/src/hyperloglog.rs@@ -4034,17 +4034,15 @@ pub(crate) const BIAS_DATA_VEC: &[&[f64]] = &[],];-use std::array;-#[derive(Clone)]pub struct HyperLogLog<const N: usize> {- registers: Box<[u8; N]>,+ registers: Vec<u8>,}impl<const N: usize> Default for HyperLogLog<N> {fn default() -> Self {Self {- registers: Box::new(array::from_fn(|_| 0)),+ registers: vec![0; N],}}}diff --git a/core/src/lib.rs b/core/src/lib.rsindex e951c7cc..fdf56824 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -37,6 +37,7 @@ mod bangs;mod collector;mod directory;mod entity_index;+mod executor;mod exponential_backoff;mod fastfield_cache;mod human_website_annotations;@@ -59,6 +60,7 @@ mod snippet;mod sonic;mod spell;mod subdomain_count;+mod summarizer;mod tokenizer;#[allow(unused)]mod ttl_cache;diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 5d414cb8..8d7188c5 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -26,7 +26,7 @@ use crate::{},webpage::Webpage,};-use std::{array, ops::Deref, sync::Arc};+use std::{ops::Deref, sync::Arc};use chrono::Utc;use tantivy::DocId;@@ -270,7 +270,7 @@ pub struct SignalAggregator {personal_centrality: Option<Arc<online_harmonic::Scorer>>,inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,field_boost: FieldBoost,- fetch_time_ms_cache: [f64; 1000],+ fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,topic_scorer: Option<topic::Scorer>,query_centrality: Option<Arc<online_harmonic::Scorer>>,@@ -299,7 +299,9 @@ impl SignalAggregator {let signal_coefficients = SignalCoefficient::new(coefficients);let field_boost = FieldBoost::new(boosts);- let fetch_time_ms_cache = array::from_fn(|fetch_time| 1.0 / (fetch_time as f64 + 1.0));+ let fetch_time_ms_cache: Vec<_> = (0..1000)+ .map(|fetch_time| 1.0 / (fetch_time as f64 + 1.0))+ .collect();let update_time_cache = (0..(3 * 365 * 24)).map(|hours_since_update| 1.0 / ((hours_since_update as f64 + 1.0).log2()))diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rsindex b0c47857..1b00d930 100644--- a/core/src/spell/mod.rs+++ b/core/src/spell/mod.rs@@ -17,6 +17,7 @@ pub mod dictionary;pub mod distance;pub mod spell_checker;pub mod splitter;+pub mod word2vec;use itertools::intersperse;use serde::{Deserialize, Serialize};diff --git a/core/src/spell/word2vec.rs b/core/src/spell/word2vec.rsnew file mode 100644index 00000000..6351c907--- /dev/null+++ b/core/src/spell/word2vec.rs@@ -0,0 +1,130 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++// Modified from: https://github.com/DimaKudosh/word2vec+use std::{collections::HashMap, io::BufRead};++use byteorder::{LittleEndian, ReadBytesExt};++#[derive(thiserror::Error, Debug)]+pub enum Error {+ #[error("Wrong header")]+ WrongHeader,++ #[error("IO")]+ IO(#[from] std::io::Error),+}++struct WordVectorReader<R: BufRead> {+ vocabulary_size: usize,+ vector_size: usize,+ reader: R,+ ended_early: bool,+ vectors_read: usize,+}++impl<R: BufRead> WordVectorReader<R> {+ pub fn vocabulary_size(&self) -> usize {+ return self.vocabulary_size;+ }++ pub fn vector_size(&self) -> usize {+ return self.vector_size;+ }++ pub fn new_from_reader(mut reader: R) -> Result<WordVectorReader<R>, Error> {+ // Read UTF8 header string from start of file+ let mut header = String::new();+ reader.read_line(&mut header)?;++ //Parse 2 integers, separated by whitespace+ let header_info = header+ .split_whitespace()+ .filter_map(|x| x.parse::<usize>().ok())+ .take(2)+ .collect::<Vec<usize>>();+ if header_info.len() != 2 {+ return Err(Error::WrongHeader);+ }++ //We've successfully read the header, ready to read vectors+ Ok(WordVectorReader {+ vocabulary_size: header_info[0],+ vector_size: header_info[1],+ vectors_read: 0,+ ended_early: false,+ reader,+ })+ }+}++impl<R: BufRead> Iterator for WordVectorReader<R> {+ type Item = (String, Vec<f32>);++ fn next(&mut self) -> Option<(String, Vec<f32>)> {+ if self.vectors_read == self.vocabulary_size {+ return None;+ }++ // Read the bytes of the word string+ let mut word_bytes: Vec<u8> = Vec::new();+ if self.reader.read_until(b' ', &mut word_bytes).is_err() {+ // End the stream if a read error occured+ self.ended_early = true;+ return None;+ }++ // trim newlines, some vector files have newlines in front of a new word, others don't+ let word = match String::from_utf8(word_bytes) {+ Err(_) => {+ self.ended_early = true;+ return None;+ }+ Ok(word) => word.trim().into(),+ };++ // Read floats of the vector+ let mut vector: Vec<f32> = Vec::with_capacity(self.vector_size);+ for _ in 0..self.vector_size {+ match self.reader.read_f32::<LittleEndian>() {+ Err(_) => {+ self.ended_early = true;+ return None;+ }+ Ok(value) => vector.push(value),+ }+ }++ self.vectors_read += 1;+ Some((word, vector))+ }+}++pub struct WordVec(Vec<f32>);+impl WordVec {+ pub(crate) fn sim(&self, passage_vec: &WordVec) -> f32 {+ todo!();+ }+}++pub struct Word2Vec {+ vectors: HashMap<String, WordVec>,+}+impl Word2Vec {+ pub(crate) fn get(&self, word: &str) -> Option<&WordVec> {+ todo!()+ }+}diff --git a/core/src/summarizer.rs b/core/src/summarizer.rsnew file mode 100644index 00000000..0b33e6b3--- /dev/null+++ b/core/src/summarizer.rs@@ -0,0 +1,119 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ cmp::{Ordering, Reverse},+ collections::BinaryHeap,+};++use itertools::intersperse;++use crate::spell::word2vec::Word2Vec;++pub struct Summarizer {+ word2vec: Word2Vec,+ top_n_passages: usize,+}++struct CandidatePassage<'a> {+ passage: &'a str,+ index: usize,+ score: f32,+}++impl<'a> PartialOrd for CandidatePassage<'a> {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.score.partial_cmp(&other.score)+ }+}++impl<'a> Ord for CandidatePassage<'a> {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(Ordering::Equal)+ }+}++impl<'a> PartialEq for CandidatePassage<'a> {+ fn eq(&self, other: &Self) -> bool {+ self.score == other.score+ }+}++impl<'a> Eq for CandidatePassage<'a> {}++impl Summarizer {+ pub fn summarize(&self, query: &str, text: &str) -> Option<String> {+ let query_vectors: Vec<_> = query+ .split_whitespace()+ .filter_map(|word| self.word2vec.get(word))+ .collect();++ if query_vectors.is_empty() {+ return None;+ }++ let mut best_passages: BinaryHeap<Reverse<CandidatePassage<'_>>> =+ BinaryHeap::with_capacity(self.top_n_passages);++ // for (index, passage) in text.split_whitespace().windows(100).enumerate() {+ for (index, passage) in text.split_whitespace().enumerate() {+ let mut score = 0.0;+ let mut count = 0;++ for passage_vec in passage+ .split_whitespace()+ .filter_map(|word| self.word2vec.get(word))+ {+ score += query_vectors+ .iter()+ .map(|vec| vec.sim(passage_vec))+ .sum::<f32>();++ count += 1;+ }++ score /= count as f32;++ let candidate = CandidatePassage {+ passage,+ index,+ score,+ };++ if best_passages.len() >= self.top_n_passages {+ if let Some(mut worst) = best_passages.peek_mut() {+ if worst.0.score < candidate.score {+ *worst = Reverse(candidate);+ }+ }+ } else {+ best_passages.push(Reverse(candidate));+ }+ }++ if best_passages.is_empty() {+ return None;+ }++ let mut best_passages: Vec<_> = best_passages.into_iter().map(|r| r.0).collect();+ best_passages.sort_by_key(|a| a.index);++ let extractive_summary: String =+ intersperse(best_passages.into_iter().map(|p| p.passage), ". ").collect();++ todo!("run abstractive summary model");+ }+}diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rsindex 5521b3b6..a4451e4d 100644--- a/core/src/tokenizer.rs+++ b/core/src/tokenizer.rs@@ -270,37 +270,62 @@ pub struct FlattenedJson {flattened_json: String,}-fn rec_flatten(val: serde_json::Value) -> Vec<String> {- match val {- serde_json::Value::Null => vec![],- serde_json::Value::Bool(b) => vec![format!("=\"{b}\"")],- serde_json::Value::Number(n) => vec![format!("=\"{n}\"")],- serde_json::Value::String(s) => vec![format!("=\"{}\"", s.replace('"', "\\\""))],- serde_json::Value::Array(arr) => arr- .into_iter()- .flat_map(|val| rec_flatten(val).into_iter())- .collect(),- serde_json::Value::Object(map) => {- let mut res = Vec::new();- for (key, value) in map {- let mut k = ".".to_string();- k.push_str(&key);-- let rec = rec_flatten(value);- if rec.is_empty() {- res.push(k);- } else {- for val in rec {- let mut k = k.clone();- k.push_str(&val);- res.push(k)- }+struct IntermediateFlatValue {+ parent_keys: Vec<String>,+ val: serde_json::Value,+}++fn flatten(val: serde_json::Value) -> Vec<String> {+ let mut res = Vec::new();++ let mut stack = Vec::new();+ stack.push(IntermediateFlatValue {+ parent_keys: Vec::new(),+ val,+ });++ while let Some(elem) = stack.pop() {+ match elem.val {+ serde_json::Value::Null => res.push(+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect(),+ ),+ serde_json::Value::Bool(b) => {+ let key: String =+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();+ res.push(format!("{key}=\"{b}\""))+ }+ serde_json::Value::Number(n) => {+ let key: String =+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();+ res.push(format!("{key}=\"{n}\""))+ }+ serde_json::Value::String(s) => {+ let key: String =+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();+ res.push(format!("{key}=\"{}\"", s.replace('"', "\\\"")))+ }+ serde_json::Value::Array(arr) => {+ for item in arr {+ stack.push(IntermediateFlatValue {+ parent_keys: elem.parent_keys.clone(),+ val: item,+ });}}+ serde_json::Value::Object(map) => {+ for (key, val) in map {+ let mut parent_keys = elem.parent_keys.clone();+ parent_keys.push(key);- res+ stack.push(IntermediateFlatValue { parent_keys, val });+ }+ }}}++ res.reverse();++ res}impl FlattenedJson {@@ -311,13 +336,8 @@ impl FlattenedJson {let json = serde_json::to_string(value)?;let val: serde_json::Value = serde_json::from_str(&json)?;- let flattened_json = itertools::intersperse(- rec_flatten(val)- .into_iter()- .map(|l| l.strip_prefix('.').unwrap().to_string()),- "\n".to_string(),- )- .collect();+ let flattened_json =+ itertools::intersperse(flatten(val).into_iter(), "\n".to_string()).collect();Ok(Self { flattened_json })}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 94f2441a..462941fc 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -27,13 +27,12 @@ use uuid::Uuid;use segment::Segment;use crate::directory::{self, DirEntry};+use crate::executor::Executor;use crate::webpage::Url;use crate::kv::rocksdb_store::RocksDbStore;pub mod centrality;-mod executor;-use self::executor::Executor;use self::segment::{CachedStore, SegmentNodeID};const MAX_NUM_SEGMENTS: usize = 400;diff --git a/core/src/webpage/schema_org/microdata.rs b/core/src/webpage/schema_org/microdata.rsindex 228e4b0a..5b887bd9 100644--- a/core/src/webpage/schema_org/microdata.rs+++ b/core/src/webpage/schema_org/microdata.rs@@ -17,7 +17,7 @@//! Almost spec compliant microdata parser: https://html.spec.whatwg.org/multipage/microdata.htmuse kuchiki::NodeRef;-use std::collections::HashMap;+use std::collections::{HashMap, VecDeque};use thiserror::Error;use super::{RawItem, RawOneOrMany, RawProperty};@@ -36,27 +36,28 @@ pub enum Error {Json(#[from] serde_json::Error),}-fn rec_text_contents(node: NodeRef) -> Vec<RawProperty> {+fn parse_properties(node: NodeRef) -> Vec<RawProperty> {let mut res = Vec::new();+ let mut q = VecDeque::new();for child in node.children() {- match child.data() {+ q.push_back(child);+ }++ while let Some(node) = q.pop_front() {+ match node.data() {kuchiki::NodeData::Element(elem) => {if elem.name.local.to_string().as_str() == "code" {let mut properties = HashMap::new();properties.insert("text".to_string(),- RawOneOrMany::One(RawProperty::String(child.text_contents())),+ RawOneOrMany::One(RawProperty::String(node.text_contents())),);res.push(RawProperty::Item(RawItem {itemtype: Some(RawOneOrMany::One("SourceCode".to_string())),properties,}))- } else if !elem.attributes.borrow().contains("itemprop")- && !elem.attributes.borrow().contains("itemscope")- {- res.append(&mut rec_text_contents(child));}}kuchiki::NodeData::Text(text) => {@@ -71,13 +72,17 @@ fn rec_text_contents(node: NodeRef) -> Vec<RawProperty> {}_ => {}}++ for child in node.children().rev() {+ q.push_front(child);+ }}res}fn text_contents(node: NodeRef) -> Vec<RawProperty> {- let properties = rec_text_contents(node);+ let properties = parse_properties(node);// merge consecutive stringslet mut res = Vec::new();