web search done right
Ftr/configure command (#65)
Mikkel Denker2c127d5f3950
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 1eeabf5d..6fc6c4ac 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -33,5 +33,4 @@ The process is simple and fast. Upon your first pull request, you will be prompt* (Optional) Install cargo-watch by running `cargo install cargo-watch`. This makes frontend development easier.* (Optional) Install just by running `cargo install just`. This allows you to run the scripts in the Justfile. A Justfile is basically a simple Makefile, so this is just for convenience.-After the non optional steps you can now run `cargo test` and should see all tests passing. If you have followed the optional steps, you should be able to run `just local` which will index some warc files and launch the frontend on `0.0.0.0:3000`.-After you have run the intial indexing, you can run `just frontend` to reuse the index.+After the non optional steps you can now run `cargo test` and should see all tests passing. If you have followed the optional steps, you should be able to run `just configure` which will setup a small local index. You can run `just frontend` to start the search server and launch the frontend at `0.0.0.0:3000`.diff --git a/Cargo.lock b/Cargo.lockindex 1ac839f4..66293c89 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -306,6 +306,15 @@ dependencies = ["crunchy",]+[[package]]+name = "block-buffer"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"+dependencies = [+ "generic-array",+]+[[package]]name = "boomphf"version = "0.5.9"@@ -538,6 +547,15 @@ version = "0.8.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"+[[package]]+name = "cpufeatures"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"+dependencies = [+ "libc",+]+[[package]]name = "crc32fast"version = "1.3.2"@@ -634,6 +652,16 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"+[[package]]+name = "crypto-mac"+version = "0.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"+dependencies = [+ "generic-array",+ "subtle",+]+[[package]]name = "cssparser"version = "0.27.2"@@ -736,6 +764,8 @@ dependencies = ["regex","reqwest","rocksdb",+ "rusoto_core",+ "rusoto_s3","rust-stemmers","serde","serde_json",@@ -747,6 +777,7 @@ dependencies = ["tower-http","tracing","tracing-subscriber",+ "urlencoding","uuid","whatlang",]@@ -779,6 +810,15 @@ version = "0.1.12"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"+[[package]]+name = "digest"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"+dependencies = [+ "generic-array",+]+[[package]]name = "dirs-next"version = "2.0.0"@@ -929,7 +969,7 @@ dependencies = ["futures-sink","nanorand","pin-project",- "spin",+ "spin 0.9.4",][[package]]@@ -1103,6 +1143,16 @@ dependencies = ["winapi",]+[[package]]+name = "generic-array"+version = "0.14.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"+dependencies = [+ "typenum",+ "version_check",+]+[[package]]name = "getrandom"version = "0.1.16"@@ -1201,6 +1251,22 @@ dependencies = ["libc",]+[[package]]+name = "hex"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"++[[package]]+name = "hmac"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"+dependencies = [+ "crypto-mac",+ "digest",+]+[[package]]name = "html-escape"version = "0.2.11"@@ -1300,6 +1366,21 @@ dependencies = ["want",]+[[package]]+name = "hyper-rustls"+version = "0.23.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac"+dependencies = [+ "http",+ "hyper",+ "log",+ "rustls",+ "rustls-native-certs",+ "tokio",+ "tokio-rustls",+]+[[package]]name = "hyper-tls"version = "0.5.0"@@ -1668,6 +1749,17 @@ version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"+[[package]]+name = "md-5"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"+dependencies = [+ "block-buffer",+ "digest",+ "opaque-debug",+]+[[package]]name = "md5"version = "0.7.0"@@ -1901,6 +1993,12 @@ version = "11.1.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"+[[package]]+name = "opaque-debug"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"+[[package]]name = "openssl"version = "0.10.40"@@ -2461,6 +2559,21 @@ dependencies = ["winreg",]+[[package]]+name = "ring"+version = "0.16.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"+dependencies = [+ "cc",+ "libc",+ "once_cell",+ "spin 0.5.2",+ "untrusted",+ "web-sys",+ "winapi",+]+[[package]]name = "rocksdb"version = "0.19.0"@@ -2471,6 +2584,88 @@ dependencies = ["librocksdb-sys",]+[[package]]+name = "rusoto_core"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"+dependencies = [+ "async-trait",+ "base64",+ "bytes",+ "crc32fast",+ "futures",+ "http",+ "hyper",+ "hyper-rustls",+ "lazy_static",+ "log",+ "rusoto_credential",+ "rusoto_signature",+ "rustc_version",+ "serde",+ "serde_json",+ "tokio",+ "xml-rs",+]++[[package]]+name = "rusoto_credential"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"+dependencies = [+ "async-trait",+ "chrono",+ "dirs-next",+ "futures",+ "hyper",+ "serde",+ "serde_json",+ "shlex",+ "tokio",+ "zeroize",+]++[[package]]+name = "rusoto_s3"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d"+dependencies = [+ "async-trait",+ "bytes",+ "futures",+ "rusoto_core",+ "xml-rs",+]++[[package]]+name = "rusoto_signature"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"+dependencies = [+ "base64",+ "bytes",+ "chrono",+ "digest",+ "futures",+ "hex",+ "hmac",+ "http",+ "hyper",+ "log",+ "md-5",+ "percent-encoding",+ "pin-project-lite",+ "rusoto_credential",+ "rustc_version",+ "serde",+ "sha2",+ "tokio",+]+[[package]]name = "rust-stemmers"version = "1.2.0"@@ -2496,6 +2691,39 @@ dependencies = ["semver",]+[[package]]+name = "rustls"+version = "0.20.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c"+dependencies = [+ "log",+ "ring",+ "sct",+ "webpki",+]++[[package]]+name = "rustls-native-certs"+version = "0.6.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"+dependencies = [+ "openssl-probe",+ "rustls-pemfile",+ "schannel",+ "security-framework",+]++[[package]]+name = "rustls-pemfile"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55"+dependencies = [+ "base64",+]+[[package]]name = "rustversion"version = "1.0.6"@@ -2545,6 +2773,16 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"+[[package]]+name = "sct"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"+dependencies = [+ "ring",+ "untrusted",+]+[[package]]name = "security-framework"version = "2.6.1"@@ -2657,6 +2895,19 @@ dependencies = ["stable_deref_trait",]+[[package]]+name = "sha2"+version = "0.9.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"+dependencies = [+ "block-buffer",+ "cfg-if",+ "cpufeatures",+ "digest",+ "opaque-debug",+]+[[package]]name = "sharded-slab"version = "0.1.4"@@ -2709,6 +2960,12 @@ dependencies = ["winapi",]+[[package]]+name = "spin"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"+[[package]]name = "spin"version = "0.9.4"@@ -2756,6 +3013,12 @@ version = "0.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"+[[package]]+name = "subtle"+version = "2.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"+[[package]]name = "syn"version = "1.0.96"@@ -3082,6 +3345,17 @@ dependencies = ["tokio",]+[[package]]+name = "tokio-rustls"+version = "0.23.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"+dependencies = [+ "rustls",+ "tokio",+ "webpki",+]+[[package]]name = "tokio-util"version = "0.6.10"@@ -3242,6 +3516,12 @@ version = "0.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"+[[package]]+name = "typenum"+version = "1.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"+[[package]]name = "unicase"version = "2.6.0"@@ -3284,6 +3564,12 @@ version = "0.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"+[[package]]+name = "untrusted"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"+[[package]]name = "url"version = "2.2.2"@@ -3296,6 +3582,12 @@ dependencies = ["percent-encoding",]+[[package]]+name = "urlencoding"+version = "2.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9"+[[package]]name = "utf-8"version = "0.7.6"@@ -3457,6 +3749,16 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "webpki"+version = "0.22.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"+dependencies = [+ "ring",+ "untrusted",+]+[[package]]name = "weezl"version = "0.1.7"@@ -3565,6 +3867,18 @@ dependencies = ["rand_core 0.6.3",]+[[package]]+name = "xml-rs"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"++[[package]]+name = "zeroize"+version = "1.5.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"+[[package]]name = "zstd-sys"version = "2.0.1+zstd.1.5.2"diff --git a/Cargo.toml b/Cargo.tomlindex 673e0e3b..3fd7c9ed 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -7,6 +7,9 @@ authors = ['Cuely ApS. <[email protected]>']# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html+[features]+dev = ["rusoto_s3", "rusoto_core"]+[[bin]]name = "cuely"@@ -60,6 +63,9 @@ base64 = "0.13.0"async-trait = "0.1.57"boomphf = { version = "0.5.9", features = ["serde"] }axum-macros = "0.2.3"+urlencoding = "2.1.2"+rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false, optional = true }+rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"], optional = true }[dev-dependencies]criterion = "0.3.6"diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex 07e46301..dc15744e 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -112,7 +112,7 @@ import { askama } from "../askama";</span>))}<span class="[&:nth-child(2)]:before:content-['—']">- {askama`item.snippet $ {{lorem.lines}}`}+ {askama`item.snippet|safe $ {{lorem.lines}}`}</span></div></div>diff --git a/justfile b/justfileindex 2d323990..b5b75f01 100644--- a/justfile+++ b/justfile@@ -1,17 +1,7 @@-@unpack-data:- rm -rf data- tar -zxvf data.tar.gz-@profile-indexer:sudo rm -rf data/indexcargo flamegraph --root -- indexer local configs/indexer/profile.toml-@worker:- cargo run --release -- webgraph worker configs/webgraph/worker.toml--@master:- cargo run --release -- webgraph master configs/webgraph/master.toml-@webgraph:rm -rf data/webgraphcargo run --release -- webgraph local configs/webgraph/local.toml@@ -30,17 +20,10 @@@astro:cd frontend; npm run dev-@local:- cd frontend; npm install- rm -rf data/index/- cargo run --release -- indexer local configs/indexer/local.toml- mv data/index/CC-MAIN-*/* data/index/- just frontend+@configure:+ cd frontend; npm install; npm run build+ cargo run --release --all-features -- configure@entity:rm -rf data/entity- cargo run --release -- indexer entity data/enwiki-20220801-pages-articles-multistream.xml.bz2 data/entity--@pack-data:- rm -f data.tar.gz- tar --exclude="data/enwiki*" --exclude="data/warc_files" --exclude="data/webgraph" -zcvf data.tar.gz data+ cargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entitydiff --git a/src/entrypoint/autosuggest_scrape.rs b/src/entrypoint/autosuggest_scrape.rsnew file mode 100644index 00000000..294cd47b--- /dev/null+++ b/src/entrypoint/autosuggest_scrape.rs@@ -0,0 +1,184 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ collections::{HashSet, VecDeque},+ path::Path,+ str::FromStr,+};++use csv::Writer;+use indicatif::{ProgressBar, ProgressStyle};++use crate::Result;++fn suggestions(query: &str, gl: &str) -> Result<Vec<String>> {+ let url = format!(+ "https://www.google.com/complete/search?q={}&gl={}&client=gws-wiz&xssi=t",+ urlencoding::encode(query),+ gl+ );++ let client = reqwest::blocking::Client::new();+ let builder = client.get(url);++ let input = builder.send()?.text()?;++ let mut input = input.split('\n');+ input.next();+ let input = input.next().expect("None option");++ let output: serde_json::Value = serde_json::from_str(input)?;+ let mut suggestions = Vec::new();++ if let serde_json::Value::Array(arr) = output {+ if let serde_json::Value::Array(arr) = arr[0].clone() {+ for result in arr {+ if let serde_json::Value::Array(result) = result {+ if let serde_json::Value::String(result) = result[0].clone() {+ let result = result.replace("<b>", "").replace("</b>", "");+ suggestions.push(result);+ }+ }+ }+ }+ }++ Ok(suggestions)+}++pub enum Gl {+ Us,+}++impl ToString for Gl {+ fn to_string(&self) -> String {+ match self {+ Gl::Us => "us".to_string(),+ }+ }+}++impl FromStr for Gl {+ type Err = crate::Error;++ fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {+ match s.to_ascii_lowercase().as_str() {+ "us" => Ok(Gl::Us),+ _ => Err(crate::Error::UnknownCLIOption),+ }+ }+}++fn save_queries<P: AsRef<Path>>(queries: &HashSet<String>, path: P) -> Result<()> {+ let mut wtr = Writer::from_path(&path)?;++ let mut queries: Vec<_> = queries.iter().collect();+ queries.sort();++ for query in queries {+ wtr.write_record([query])?;+ }++ wtr.flush()?;++ Ok(())+}++pub fn run<P: AsRef<Path>>(+ queries_to_scrape: usize,+ gl: Gl,+ ms_sleep_between_req: u64,+ output_dir: P,+) -> Result<()> {+ let mut queries = HashSet::new();+ let mut queue = VecDeque::new();++ let pb = ProgressBar::new(queries_to_scrape as u64);+ pb.set_style(+ ProgressStyle::default_bar()+ .template("{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})")+ .progress_chars("#>-"),+ );++ for c in 'a'..='z' {+ queue.push_back(c.to_string());+ }++ let path = output_dir+ .as_ref()+ .join(format!("queries_{:}.csv", gl.to_string().as_str()));++ let mut queries_since_last_save = 0;++ while let Some(query) = queue.pop_front() {+ let res = suggestions(&query, gl.to_string().as_str());++ if res.is_err() {+ continue;+ }++ let res = res.unwrap();+ let mut new_queries = 0;++ for next_query in res {+ if queries.contains(&next_query) {+ continue;+ }++ let mut new = Vec::new();+ for c in next_query.chars() {+ let q = new.clone().into_iter().collect();+ new.push(c);++ if !queries.contains(&q) {+ queue.push_back(q);+ }+ }++ for q in next_query.split_whitespace() {+ if !queries.contains(q) {+ queries.insert(q.to_string());+ queue.push_back(q.to_string());+ }+ }++ new_queries += 1;+ queries.insert(next_query);+ pb.tick();+ pb.set_position(queries.len() as u64);+ }++ if queries.len() >= queries_to_scrape {+ break;+ }++ queries_since_last_save += new_queries;++ if queries_since_last_save > 1_000 {+ save_queries(&queries, &path)?;+ queries_since_last_save = 0;+ }++ std::thread::sleep(std::time::Duration::from_millis(ms_sleep_between_req));+ }++ pb.finish();++ save_queries(&queries, &path)?;++ Ok(())+}diff --git a/src/entrypoint/configure.rs b/src/entrypoint/configure.rsnew file mode 100644index 00000000..95556fea--- /dev/null+++ b/src/entrypoint/configure.rs@@ -0,0 +1,186 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use rusoto_core::credential::StaticProvider;+use rusoto_core::HttpClient;+use rusoto_s3::{GetObjectRequest, S3Client, S3};+use tokio::fs::File;+use tokio::io;+use tracing::{debug, info};++use crate::entrypoint::indexer;+use crate::Result;+use std::fs::{self};+use std::path::Path;++use super::{webgraph, Centrality, EntityIndexer};++const DATA_PATH: &str = "data";+const BUCKET_NAME: &str = "s3.cuely.io";++fn download_files() {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async {+ let creds = StaticProvider::new("".to_string(), "".to_string(), None, None);+ let client = S3Client::new_with(+ HttpClient::new().unwrap(),+ creds,+ rusoto_core::Region::EuCentral1,+ );++ for name in [+ "enwiki_subset.xml.bz2",+ "queries_us.csv",+ "sample.warc.gz",+ "bangs.json",+ ] {+ info!("Downloading {}", name);+ let mut object = client+ .get_object(GetObjectRequest {+ bucket: BUCKET_NAME.into(),+ key: name.into(),+ ..Default::default()+ })+ .await+ .unwrap();++ let body = object.body.take().expect("The object has no body");++ let mut body = body.into_async_read();+ let mut file = File::create(Path::new(DATA_PATH).join(name)).await.unwrap();+ io::copy(&mut body, &mut file).await.unwrap();+ }+ });+}++fn create_webgraph() -> Result<()> {+ debug!("Creating webgraph");+ let out_path_tmp = Path::new(DATA_PATH).join("webgraph_tmp");+ let out_path = Path::new(DATA_PATH).join("webgraph");++ if out_path_tmp.exists() {+ std::fs::remove_dir_all(&out_path_tmp)?;+ }+ if out_path.exists() {+ std::fs::remove_dir_all(&out_path)?;+ }++ let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");++ let job = webgraph::Job {+ config: webgraph::JobConfig::Local(crate::LocalConfig {+ folder: ".".to_string(),+ names: vec![warc_path.to_str().unwrap().to_string()],+ }),+ warc_paths: vec![warc_path.to_str().unwrap().to_string()],+ graph_base_path: out_path_tmp.to_str().unwrap().to_string(),+ };++ let graph = webgraph::process_job(&job);+ std::fs::rename(graph.path, out_path)?;+ std::fs::remove_dir_all(&out_path_tmp)?;++ Ok(())+}++fn calculate_centrality() {+ debug!("Calculating centrality");+ let webgraph_path = Path::new(DATA_PATH).join("webgraph");+ let out_path = Path::new(DATA_PATH).join("centrality");+ Centrality::run(webgraph_path, out_path);+}++fn create_inverted_index() -> Result<()> {+ debug!("Creating inverted index");+ let out_path = Path::new(DATA_PATH).join("index");+ let out_path_tmp = Path::new(DATA_PATH).join("index_tmp");++ if out_path_tmp.exists() {+ std::fs::remove_dir_all(&out_path_tmp)?;+ }+ if out_path.exists() {+ std::fs::remove_dir_all(&out_path)?;+ }++ let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");++ let job = indexer::Job {+ source_config: indexer::JobConfig::Local(crate::LocalConfig {+ folder: ".".to_string(),+ names: vec![warc_path.to_str().unwrap().to_string()],+ }),+ download_images: false,+ warc_paths: vec![warc_path.to_str().unwrap().to_string()],+ base_path: out_path_tmp.to_str().unwrap().to_string(),+ host_centrality_threshold: None,+ max_num_segments: 1,+ };++ let webgraph_path = Path::new(DATA_PATH).join("webgraph");+ let centrality_path = Path::new(DATA_PATH).join("centrality");++ let worker = indexer::IndexingWorker::new(+ centrality_path.to_str().unwrap().to_string(),+ Some(webgraph_path.to_str().unwrap().to_string()),+ );++ let index = indexer::process_job(&job, &worker);+ std::fs::rename(index.path, out_path)?;+ std::fs::remove_dir_all(&out_path_tmp)?;++ Ok(())+}++fn create_entity_index() -> Result<()> {+ let out_path = Path::new(DATA_PATH).join("entity");+ if out_path.exists() {+ std::fs::remove_dir_all(&out_path)?;+ }++ let wiki_path = Path::new(DATA_PATH).join("enwiki_subset.xml.bz2");++ EntityIndexer::run(+ wiki_path.to_str().unwrap().to_string(),+ out_path.to_str().unwrap().to_string(),+ )?;++ Ok(())+}++fn index_files() -> Result<()> {+ create_webgraph()?;+ calculate_centrality();+ create_inverted_index()?;+ create_entity_index()?;++ Ok(())+}++pub fn run() -> Result<()> {+ let p = Path::new(DATA_PATH);++ if !p.exists() {+ fs::create_dir_all(p)?;+ }++ download_files();+ index_files()?;++ Ok(())+}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex af1229f8..73ec6f04 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -38,28 +38,28 @@ use crate::{pub struct Indexer {}#[derive(Debug, Serialize, Deserialize, Clone)]-enum JobConfig {+pub enum JobConfig {Http(HttpConfig),Local(LocalConfig),}#[derive(Debug, Serialize, Deserialize)]-struct Job {- source_config: JobConfig,- download_images: bool,- warc_paths: Vec<String>,- base_path: String,- host_centrality_threshold: Option<f64>,- max_num_segments: u32,+pub struct Job {+ pub source_config: JobConfig,+ pub download_images: bool,+ pub warc_paths: Vec<String>,+ pub base_path: String,+ pub host_centrality_threshold: Option<f64>,+ pub max_num_segments: u32,}-struct IndexingWorker {+pub struct IndexingWorker {centrality_store: CentralityStore,webgraph: Option<Webgraph>,}impl IndexingWorker {- fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {+ pub fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {Self {centrality_store: CentralityStore::open(centrality_store_path),webgraph: webgraph_path.map(|path| {@@ -202,7 +202,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {index}-fn process_job(job: &Job, worker: &IndexingWorker) -> Index {+pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {tokio::runtime::Builder::new_current_thread().enable_all().build()diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex 9ab62d1f..549241f3 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -13,7 +13,10 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod autosuggest_scrape;mod centrality;+#[cfg(feature = "dev")]+pub mod configure;mod entity;pub mod frontend;pub mod indexer;diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 5ae7f73f..0494397f 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -34,16 +34,16 @@ use tracing::{info, trace};struct GraphPointer(String);#[derive(Debug, Serialize, Deserialize, Clone)]-enum JobConfig {+pub enum JobConfig {Http(HttpConfig),Local(LocalConfig),}#[derive(Debug, Serialize, Deserialize)]-struct Job {- config: JobConfig,- warc_paths: Vec<String>,- graph_base_path: String,+pub struct Job {+ pub config: JobConfig,+ pub warc_paths: Vec<String>,+ pub graph_base_path: String,}fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {@@ -105,7 +105,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {graph}-fn process_job(job: &Job) -> webgraph::Webgraph {+pub fn process_job(job: &Job) -> webgraph::Webgraph {tokio::runtime::Builder::new_current_thread().enable_all().build()diff --git a/src/image_downloader.rs b/src/image_downloader.rsindex 344ccfa0..97e56fd5 100644--- a/src/image_downloader.rs+++ b/src/image_downloader.rs@@ -44,7 +44,7 @@ wherecontinue;}- for duration in ExponentialBackoff::from_millis(10).take(5) {+ for duration in ExponentialBackoff::from_millis(10).take(4) {if let Some(image) = url.download_bytes(self.timeout.unwrap_or_else(|| Duration::from_secs(20))).await@@ -123,17 +123,25 @@ where}async fn download_pending_images_async(&mut self, store: &mut impl ImageStore<K>) {- let results = futures::stream::iter(+ let mut stream = futures::stream::iter(self.image_download_jobs.drain().map(|job| async move { job.download().await }),)- .buffer_unordered(20)- .collect::<Vec<Option<DownloadedImage<K>>>>()- .await;+ .buffer_unordered(20);- for result in results.into_iter().flatten() {- store.insert(result.key, result.image);+ let mut inserts_since_flush = 0;++ while let Some(result) = stream.next().await {+ if let Some(result) = result {+ store.insert(result.key, result.image);+ inserts_since_flush += 1;+ }++ if inserts_since_flush > 1_000 {+ store.flush();+ inserts_since_flush = 0;+ }}store.flush();diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 8a1adbe0..8b79f4a0 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -34,6 +34,7 @@ use crate::webpage::region::Region;use crate::webpage::{StoredPrimaryImage, Webpage};use crate::Result;use crate::{schema::create_schema, tokenizer::Tokenizer};+use std::collections::HashSet;use std::fs;use std::path::Path;use std::sync::{Arc, Weak};@@ -306,50 +307,60 @@ impl InvertedIndex {}pub fn merge(mut self, mut other: InvertedIndex) -> Self {- other.commit().expect("failed to commit index");- self.commit().expect("failed to commit index");+ let path = self.path.clone();- let other_meta = other- .tantivy_index- .load_metas()- .expect("failed to laod tantivy metadata for index");+ {+ other.commit().expect("failed to commit index");+ self.commit().expect("failed to commit index");- let mut meta = self- .tantivy_index- .load_metas()- .expect("failed to laod tantivy metadata for index");+ let other_meta = other+ .tantivy_index+ .load_metas()+ .expect("failed to load tantivy metadata for index");- let x = other.path.clone();- let other_path = Path::new(x.as_str());- other.writer.wait_merging_threads().unwrap();+ let mut meta = self+ .tantivy_index+ .load_metas()+ .expect("failed to load tantivy metadata for index");- let path = self.path.clone();- let self_path = Path::new(path.as_str());- self.writer.wait_merging_threads().unwrap();-- for segment in other_meta.segments {- // TODO: handle case where current index has segment with same name- for file in segment.list_files() {- let p = other_path.join(&file);- if p.exists() {- fs::rename(p, self_path.join(&file)).unwrap();+ let x = other.path.clone();+ let other_path = Path::new(x.as_str());+ other.writer.wait_merging_threads().unwrap();++ let path = self.path.clone();+ let self_path = Path::new(path.as_str());+ self.writer.wait_merging_threads().unwrap();++ let ids: HashSet<_> = meta.segments.iter().map(|segment| segment.id()).collect();++ for segment in other_meta.segments {+ if ids.contains(&segment.id()) {+ continue;}++ // TODO: handle case where current index has segment with same name+ for file in segment.list_files() {+ let p = other_path.join(&file);+ if p.exists() {+ fs::rename(p, self_path.join(&file)).unwrap();+ }+ }+ meta.segments.push(segment);}- meta.segments.push(segment);- }- meta.segments- .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));+ meta.segments+ .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));- fs::remove_dir_all(other_path).unwrap();+ fs::remove_dir_all(other_path).unwrap();- let self_path = Path::new(&path);+ let self_path = Path::new(&path);- std::fs::write(- self_path.join("meta.json"),- serde_json::to_string_pretty(&meta).unwrap(),- )- .unwrap();+ std::fs::write(+ self_path.join("meta.json"),+ serde_json::to_string_pretty(&meta).unwrap(),+ )+ .unwrap();+ }Self::open(path).expect("failed to open index")}diff --git a/src/lib.rs b/src/lib.rsindex a89d57d0..0eca3671 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -132,8 +132,8 @@ impl WarcSource {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct LocalConfig {- folder: String,- names: Vec<String>,+ pub folder: String,+ pub names: Vec<String>,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -222,6 +222,12 @@ pub enum Error {#[error("Could not open inverted-index directory")]Directory(#[from] tantivy::directory::error::OpenDirectoryError),++ #[error("Could not convert to/from JSON")]+ Json(#[from] serde_json::Error),++ #[error("Unknown CLI option")]+ UnknownCLIOption,}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/main.rs b/src/main.rsindex aa0dfaab..f780f45d 100644--- a/src/main.rs+++ b/src/main.rs@@ -15,6 +15,9 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::Result;use clap::{Parser, Subcommand};+use cuely::entrypoint::autosuggest_scrape::{self, Gl};+#[cfg(feature = "dev")]+use cuely::entrypoint::configure;use cuely::entrypoint::indexer::IndexPointer;use cuely::entrypoint::{self, frontend, search_server};use cuely::{FrontendConfig, SearchServerConfig};@@ -52,6 +55,14 @@ enum Commands {Frontend {config_path: String,},+ AutosuggestScrape {+ num_queries: usize,+ gl: Gl,+ ms_sleep_between_req: u64,+ output_dir: String,+ },+ #[cfg(feature = "dev")]+ Configure,}#[derive(Subcommand)]@@ -163,6 +174,16 @@ fn main() -> Result<()> {.build()?.block_on(search_server::run(config))?}+ Commands::AutosuggestScrape {+ num_queries: queries_to_scrape,+ gl,+ ms_sleep_between_req,+ output_dir,+ } => {+ autosuggest_scrape::run(queries_to_scrape, gl, ms_sleep_between_req, output_dir)?;+ }+ #[cfg(feature = "dev")]+ Commands::Configure => configure::run()?,}Ok(())diff --git a/src/warc.rs b/src/warc.rsindex eecb7b01..a709178c 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -25,7 +25,7 @@ use std::time::Duration;use flate2::read::MultiGzDecoder;use futures::StreamExt;use tokio::time::sleep;-use tracing::debug;+use tracing::{debug, trace};pub(crate) struct WarcFile {bytes: Vec<u8>,@@ -108,6 +108,8 @@ impl WarcFile {if res.is_ok() {return Ok(());+ } else {+ trace!("Error {:?}", res);}debug!("warc download failed: {:?}", res.err().unwrap());diff --git a/src/webgraph/centrality/betweenness.rs b/src/webgraph/centrality/betweenness.rsindex b43b1243..f02622bf 100644--- a/src/webgraph/centrality/betweenness.rs+++ b/src/webgraph/centrality/betweenness.rs@@ -127,7 +127,7 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<N}if let Some(pb) = &pb {- pb.finish();+ pb.finish_and_clear();}let n = n as f64;