Skip to content

web search done right

Unravel recursive functions to avoid stackoverflow. Summarizer is also WIP (there might be some 'never used' warnings when compiling)

Mikkel Denker785e610db8ed
diff --git a/Cargo.lock b/Cargo.lock
index fc0f7067..ba9999e0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -788,6 +788,7 @@ dependencies = [
"base64 0.13.1",
"bincode",
"bitvec",
+ "byteorder",
"bzip2",
"chrono",
"clap 3.2.23",
diff --git a/Cargo.toml b/Cargo.toml
index a8dc6b5a..8ecaf7ee 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -66,4 +66,5 @@ rand = "0.8.5"
crossbeam-channel = "0.5.6"
num_cpus = "1.15.0"
onnxruntime = { git = "https://github.com/Cuely/onnxruntime-rs" }
-tokenizers = "0.13.2"
\ No newline at end of file
+tokenizers = "0.13.2"
+byteorder = "1.4.3"
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index ba9a28e5..06b1e910 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -72,6 +72,7 @@ num_cpus = { workspace = true }
onnxruntime = { workspace = true }
tokenizers = { workspace = true }
lalrpop-util = { workspace = true }
+byteorder = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/benches/online-harmonic.rs b/core/benches/online-harmonic.rs
index d9e9c4ac..21c04c36 100644
--- a/core/benches/online-harmonic.rs
+++ b/core/benches/online-harmonic.rs
@@ -15,7 +15,9 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let harmonic_centrality_store = HarmonicCentralityStore::open(cuely::gen_temp_path());
for (node, centrality) in centrality.host {
- harmonic_centrality_store.host.insert(node.name, centrality);
+ harmonic_centrality_store
+ .host
+ .insert(webgraph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 8fe79ea7..496b959d 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -18,12 +18,12 @@ use std::net::SocketAddr;
use std::path::Path;
use itertools::Itertools;
-use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use tokio::pin;
use tracing::{debug, info, trace};
use crate::entrypoint::async_download_all_warc_files;
+use crate::executor::Executor;
use crate::index::{FrozenIndex, Index};
use crate::mapreduce::{Manager, Map, Reduce, Worker};
use crate::ranking::centrality_store::IndexerCentralityStore;
@@ -390,28 +390,30 @@ impl Indexer {
config.topics_path.clone(),
);
- let indexes: Vec<_> = warc_paths
- .into_iter()
- .skip(config.skip_num_warc_files.unwrap_or(0))
- .take(config.limit_warc_files.unwrap_or(usize::MAX))
- .chunks(config.batch_size.unwrap_or(1))
- .into_iter()
- .map(|warc_paths| Job {
- source_config: job_config.clone(),
- warc_paths: warc_paths.collect_vec(),
- download_images: config.download_images.unwrap_or(true),
- host_centrality_threshold: config.host_centrality_threshold,
- base_path: config
- .output_path
- .clone()
- .unwrap_or_else(|| "data/index".to_string()),
- max_num_segments: config.final_num_segments.unwrap_or(20),
- })
- .collect_vec()
- .into_par_iter()
- .panic_fuse()
- .map(|job| -> IndexPointer { job.map(&worker) })
- .collect();
+ let executor = Executor::multi_thread("indexer").unwrap();
+
+ let indexes = executor
+ .map(
+ |job| -> IndexPointer { job.map(&worker) },
+ warc_paths
+ .into_iter()
+ .skip(config.skip_num_warc_files.unwrap_or(0))
+ .take(config.limit_warc_files.unwrap_or(usize::MAX))
+ .chunks(config.batch_size.unwrap_or(1))
+ .into_iter()
+ .map(|warc_paths| Job {
+ source_config: job_config.clone(),
+ warc_paths: warc_paths.collect_vec(),
+ download_images: config.download_images.unwrap_or(true),
+ host_centrality_threshold: config.host_centrality_threshold,
+ base_path: config
+ .output_path
+ .clone()
+ .unwrap_or_else(|| "data/index".to_string()),
+ max_num_segments: config.final_num_segments.unwrap_or(20),
+ }),
+ )
+ .unwrap_or_default();
Self::merge(indexes, config.final_num_segments.unwrap_or(20))?;
Ok(())
diff --git a/core/src/webgraph/executor.rs b/core/src/executor.rs
similarity index 99%
rename from core/src/webgraph/executor.rs
rename to core/src/executor.rs
index 81750750..b3300c88 100644
--- a/core/src/webgraph/executor.rs
+++ b/core/src/executor.rs
@@ -38,7 +38,7 @@ impl Executor {
pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {
let pool = ThreadPoolBuilder::new()
.num_threads(num_threads)
- .stack_size(8_000_000)
+ .stack_size(80_000_000)
.thread_name(move |num| format!("{prefix}{num}"))
.build()?;
Ok(Executor::ThreadPool(pool))
diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rs
index 9ecc8f75..7846fdb3 100644
--- a/core/src/hyperloglog.rs
+++ b/core/src/hyperloglog.rs
@@ -4034,17 +4034,15 @@ pub(crate) const BIAS_DATA_VEC: &[&[f64]] = &[
],
];
-use std::array;
-
#[derive(Clone)]
pub struct HyperLogLog<const N: usize> {
- registers: Box<[u8; N]>,
+ registers: Vec<u8>,
}
impl<const N: usize> Default for HyperLogLog<N> {
fn default() -> Self {
Self {
- registers: Box::new(array::from_fn(|_| 0)),
+ registers: vec![0; N],
}
}
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index e951c7cc..fdf56824 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -37,6 +37,7 @@ mod bangs;
mod collector;
mod directory;
mod entity_index;
+mod executor;
mod exponential_backoff;
mod fastfield_cache;
mod human_website_annotations;
@@ -59,6 +60,7 @@ mod snippet;
mod sonic;
mod spell;
mod subdomain_count;
+mod summarizer;
mod tokenizer;
#[allow(unused)]
mod ttl_cache;
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 5d414cb8..8d7188c5 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -26,7 +26,7 @@ use crate::{
},
webpage::Webpage,
};
-use std::{array, ops::Deref, sync::Arc};
+use std::{ops::Deref, sync::Arc};
use chrono::Utc;
use tantivy::DocId;
@@ -270,7 +270,7 @@ pub struct SignalAggregator {
personal_centrality: Option<Arc<online_harmonic::Scorer>>,
inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,
field_boost: FieldBoost,
- fetch_time_ms_cache: [f64; 1000],
+ fetch_time_ms_cache: Vec<f64>,
update_time_cache: Vec<f64>,
topic_scorer: Option<topic::Scorer>,
query_centrality: Option<Arc<online_harmonic::Scorer>>,
@@ -299,7 +299,9 @@ impl SignalAggregator {
let signal_coefficients = SignalCoefficient::new(coefficients);
let field_boost = FieldBoost::new(boosts);
- let fetch_time_ms_cache = array::from_fn(|fetch_time| 1.0 / (fetch_time as f64 + 1.0));
+ let fetch_time_ms_cache: Vec<_> = (0..1000)
+ .map(|fetch_time| 1.0 / (fetch_time as f64 + 1.0))
+ .collect();
let update_time_cache = (0..(3 * 365 * 24))
.map(|hours_since_update| 1.0 / ((hours_since_update as f64 + 1.0).log2()))
diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rs
index b0c47857..1b00d930 100644
--- a/core/src/spell/mod.rs
+++ b/core/src/spell/mod.rs
@@ -17,6 +17,7 @@ pub mod dictionary;
pub mod distance;
pub mod spell_checker;
pub mod splitter;
+pub mod word2vec;
use itertools::intersperse;
use serde::{Deserialize, Serialize};
diff --git a/core/src/spell/word2vec.rs b/core/src/spell/word2vec.rs
new file mode 100644
index 00000000..6351c907
--- /dev/null
+++ b/core/src/spell/word2vec.rs
@@ -0,0 +1,130 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// Modified from: https://github.com/DimaKudosh/word2vec
+use std::{collections::HashMap, io::BufRead};
+
+use byteorder::{LittleEndian, ReadBytesExt};
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+ #[error("Wrong header")]
+ WrongHeader,
+
+ #[error("IO")]
+ IO(#[from] std::io::Error),
+}
+
+struct WordVectorReader<R: BufRead> {
+ vocabulary_size: usize,
+ vector_size: usize,
+ reader: R,
+ ended_early: bool,
+ vectors_read: usize,
+}
+
+impl<R: BufRead> WordVectorReader<R> {
+ pub fn vocabulary_size(&self) -> usize {
+ return self.vocabulary_size;
+ }
+
+ pub fn vector_size(&self) -> usize {
+ return self.vector_size;
+ }
+
+ pub fn new_from_reader(mut reader: R) -> Result<WordVectorReader<R>, Error> {
+ // Read UTF8 header string from start of file
+ let mut header = String::new();
+ reader.read_line(&mut header)?;
+
+ //Parse 2 integers, separated by whitespace
+ let header_info = header
+ .split_whitespace()
+ .filter_map(|x| x.parse::<usize>().ok())
+ .take(2)
+ .collect::<Vec<usize>>();
+ if header_info.len() != 2 {
+ return Err(Error::WrongHeader);
+ }
+
+ //We've successfully read the header, ready to read vectors
+ Ok(WordVectorReader {
+ vocabulary_size: header_info[0],
+ vector_size: header_info[1],
+ vectors_read: 0,
+ ended_early: false,
+ reader,
+ })
+ }
+}
+
+impl<R: BufRead> Iterator for WordVectorReader<R> {
+ type Item = (String, Vec<f32>);
+
+ fn next(&mut self) -> Option<(String, Vec<f32>)> {
+ if self.vectors_read == self.vocabulary_size {
+ return None;
+ }
+
+ // Read the bytes of the word string
+ let mut word_bytes: Vec<u8> = Vec::new();
+ if self.reader.read_until(b' ', &mut word_bytes).is_err() {
+ // End the stream if a read error occured
+ self.ended_early = true;
+ return None;
+ }
+
+ // trim newlines, some vector files have newlines in front of a new word, others don't
+ let word = match String::from_utf8(word_bytes) {
+ Err(_) => {
+ self.ended_early = true;
+ return None;
+ }
+ Ok(word) => word.trim().into(),
+ };
+
+ // Read floats of the vector
+ let mut vector: Vec<f32> = Vec::with_capacity(self.vector_size);
+ for _ in 0..self.vector_size {
+ match self.reader.read_f32::<LittleEndian>() {
+ Err(_) => {
+ self.ended_early = true;
+ return None;
+ }
+ Ok(value) => vector.push(value),
+ }
+ }
+
+ self.vectors_read += 1;
+ Some((word, vector))
+ }
+}
+
+pub struct WordVec(Vec<f32>);
+impl WordVec {
+ pub(crate) fn sim(&self, passage_vec: &WordVec) -> f32 {
+ todo!();
+ }
+}
+
+pub struct Word2Vec {
+ vectors: HashMap<String, WordVec>,
+}
+impl Word2Vec {
+ pub(crate) fn get(&self, word: &str) -> Option<&WordVec> {
+ todo!()
+ }
+}
diff --git a/core/src/summarizer.rs b/core/src/summarizer.rs
new file mode 100644
index 00000000..0b33e6b3
--- /dev/null
+++ b/core/src/summarizer.rs
@@ -0,0 +1,119 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ cmp::{Ordering, Reverse},
+ collections::BinaryHeap,
+};
+
+use itertools::intersperse;
+
+use crate::spell::word2vec::Word2Vec;
+
+pub struct Summarizer {
+ word2vec: Word2Vec,
+ top_n_passages: usize,
+}
+
+struct CandidatePassage<'a> {
+ passage: &'a str,
+ index: usize,
+ score: f32,
+}
+
+impl<'a> PartialOrd for CandidatePassage<'a> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.score.partial_cmp(&other.score)
+ }
+}
+
+impl<'a> Ord for CandidatePassage<'a> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.partial_cmp(other).unwrap_or(Ordering::Equal)
+ }
+}
+
+impl<'a> PartialEq for CandidatePassage<'a> {
+ fn eq(&self, other: &Self) -> bool {
+ self.score == other.score
+ }
+}
+
+impl<'a> Eq for CandidatePassage<'a> {}
+
+impl Summarizer {
+ pub fn summarize(&self, query: &str, text: &str) -> Option<String> {
+ let query_vectors: Vec<_> = query
+ .split_whitespace()
+ .filter_map(|word| self.word2vec.get(word))
+ .collect();
+
+ if query_vectors.is_empty() {
+ return None;
+ }
+
+ let mut best_passages: BinaryHeap<Reverse<CandidatePassage<'_>>> =
+ BinaryHeap::with_capacity(self.top_n_passages);
+
+ // for (index, passage) in text.split_whitespace().windows(100).enumerate() {
+ for (index, passage) in text.split_whitespace().enumerate() {
+ let mut score = 0.0;
+ let mut count = 0;
+
+ for passage_vec in passage
+ .split_whitespace()
+ .filter_map(|word| self.word2vec.get(word))
+ {
+ score += query_vectors
+ .iter()
+ .map(|vec| vec.sim(passage_vec))
+ .sum::<f32>();
+
+ count += 1;
+ }
+
+ score /= count as f32;
+
+ let candidate = CandidatePassage {
+ passage,
+ index,
+ score,
+ };
+
+ if best_passages.len() >= self.top_n_passages {
+ if let Some(mut worst) = best_passages.peek_mut() {
+ if worst.0.score < candidate.score {
+ *worst = Reverse(candidate);
+ }
+ }
+ } else {
+ best_passages.push(Reverse(candidate));
+ }
+ }
+
+ if best_passages.is_empty() {
+ return None;
+ }
+
+ let mut best_passages: Vec<_> = best_passages.into_iter().map(|r| r.0).collect();
+ best_passages.sort_by_key(|a| a.index);
+
+ let extractive_summary: String =
+ intersperse(best_passages.into_iter().map(|p| p.passage), ". ").collect();
+
+ todo!("run abstractive summary model");
+ }
+}
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
index 5521b3b6..a4451e4d 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer.rs
@@ -270,37 +270,62 @@ pub struct FlattenedJson {
flattened_json: String,
}
-fn rec_flatten(val: serde_json::Value) -> Vec<String> {
- match val {
- serde_json::Value::Null => vec![],
- serde_json::Value::Bool(b) => vec![format!("=\"{b}\"")],
- serde_json::Value::Number(n) => vec![format!("=\"{n}\"")],
- serde_json::Value::String(s) => vec![format!("=\"{}\"", s.replace('"', "\\\""))],
- serde_json::Value::Array(arr) => arr
- .into_iter()
- .flat_map(|val| rec_flatten(val).into_iter())
- .collect(),
- serde_json::Value::Object(map) => {
- let mut res = Vec::new();
- for (key, value) in map {
- let mut k = ".".to_string();
- k.push_str(&key);
-
- let rec = rec_flatten(value);
- if rec.is_empty() {
- res.push(k);
- } else {
- for val in rec {
- let mut k = k.clone();
- k.push_str(&val);
- res.push(k)
- }
+struct IntermediateFlatValue {
+ parent_keys: Vec<String>,
+ val: serde_json::Value,
+}
+
+fn flatten(val: serde_json::Value) -> Vec<String> {
+ let mut res = Vec::new();
+
+ let mut stack = Vec::new();
+ stack.push(IntermediateFlatValue {
+ parent_keys: Vec::new(),
+ val,
+ });
+
+ while let Some(elem) = stack.pop() {
+ match elem.val {
+ serde_json::Value::Null => res.push(
+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect(),
+ ),
+ serde_json::Value::Bool(b) => {
+ let key: String =
+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();
+ res.push(format!("{key}=\"{b}\""))
+ }
+ serde_json::Value::Number(n) => {
+ let key: String =
+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();
+ res.push(format!("{key}=\"{n}\""))
+ }
+ serde_json::Value::String(s) => {
+ let key: String =
+ itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();
+ res.push(format!("{key}=\"{}\"", s.replace('"', "\\\"")))
+ }
+ serde_json::Value::Array(arr) => {
+ for item in arr {
+ stack.push(IntermediateFlatValue {
+ parent_keys: elem.parent_keys.clone(),
+ val: item,
+ });
}
}
+ serde_json::Value::Object(map) => {
+ for (key, val) in map {
+ let mut parent_keys = elem.parent_keys.clone();
+ parent_keys.push(key);
- res
+ stack.push(IntermediateFlatValue { parent_keys, val });
+ }
+ }
}
}
+
+ res.reverse();
+
+ res
}
impl FlattenedJson {
@@ -311,13 +336,8 @@ impl FlattenedJson {
let json = serde_json::to_string(value)?;
let val: serde_json::Value = serde_json::from_str(&json)?;
- let flattened_json = itertools::intersperse(
- rec_flatten(val)
- .into_iter()
- .map(|l| l.strip_prefix('.').unwrap().to_string()),
- "\n".to_string(),
- )
- .collect();
+ let flattened_json =
+ itertools::intersperse(flatten(val).into_iter(), "\n".to_string()).collect();
Ok(Self { flattened_json })
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 94f2441a..462941fc 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -27,13 +27,12 @@ use uuid::Uuid;
use segment::Segment;
use crate::directory::{self, DirEntry};
+use crate::executor::Executor;
use crate::webpage::Url;
use crate::kv::rocksdb_store::RocksDbStore;
pub mod centrality;
-mod executor;
-use self::executor::Executor;
use self::segment::{CachedStore, SegmentNodeID};
const MAX_NUM_SEGMENTS: usize = 400;
diff --git a/core/src/webpage/schema_org/microdata.rs b/core/src/webpage/schema_org/microdata.rs
index 228e4b0a..5b887bd9 100644
--- a/core/src/webpage/schema_org/microdata.rs
+++ b/core/src/webpage/schema_org/microdata.rs
@@ -17,7 +17,7 @@
//! Almost spec compliant microdata parser: https://html.spec.whatwg.org/multipage/microdata.htm
use kuchiki::NodeRef;
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
use thiserror::Error;
use super::{RawItem, RawOneOrMany, RawProperty};
@@ -36,27 +36,28 @@ pub enum Error {
Json(#[from] serde_json::Error),
}
-fn rec_text_contents(node: NodeRef) -> Vec<RawProperty> {
+fn parse_properties(node: NodeRef) -> Vec<RawProperty> {
let mut res = Vec::new();
+ let mut q = VecDeque::new();
for child in node.children() {
- match child.data() {
+ q.push_back(child);
+ }
+
+ while let Some(node) = q.pop_front() {
+ match node.data() {
kuchiki::NodeData::Element(elem) => {
if elem.name.local.to_string().as_str() == "code" {
let mut properties = HashMap::new();
properties.insert(
"text".to_string(),
- RawOneOrMany::One(RawProperty::String(child.text_contents())),
+ RawOneOrMany::One(RawProperty::String(node.text_contents())),
);
res.push(RawProperty::Item(RawItem {
itemtype: Some(RawOneOrMany::One("SourceCode".to_string())),
properties,
}))
- } else if !elem.attributes.borrow().contains("itemprop")
- && !elem.attributes.borrow().contains("itemscope")
- {
- res.append(&mut rec_text_contents(child));
}
}
kuchiki::NodeData::Text(text) => {
@@ -71,13 +72,17 @@ fn rec_text_contents(node: NodeRef) -> Vec<RawProperty> {
}
_ => {}
}
+
+ for child in node.children().rev() {
+ q.push_front(child);
+ }
}
res
}
fn text_contents(node: NodeRef) -> Vec<RawProperty> {
- let properties = rec_text_contents(node);
+ let properties = parse_properties(node);
// merge consecutive strings
let mut res = Vec::new();