Skip to content

web search done right

Ftr/configure command (#65)

Mikkel Denker2c127d5f3950
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1eeabf5d..6fc6c4ac 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -33,5 +33,4 @@ The process is simple and fast. Upon your first pull request, you will be prompt
* (Optional) Install cargo-watch by running `cargo install cargo-watch`. This makes frontend development easier.
* (Optional) Install just by running `cargo install just`. This allows you to run the scripts in the Justfile. A Justfile is basically a simple Makefile, so this is just for convenience.
-After the non optional steps you can now run `cargo test` and should see all tests passing. If you have followed the optional steps, you should be able to run `just local` which will index some warc files and launch the frontend on `0.0.0.0:3000`.
-After you have run the intial indexing, you can run `just frontend` to reuse the index.
+After the non optional steps you can now run `cargo test` and should see all tests passing. If you have followed the optional steps, you should be able to run `just configure` which will setup a small local index. You can run `just frontend` to start the search server and launch the frontend at `0.0.0.0:3000`.
diff --git a/Cargo.lock b/Cargo.lock
index 1ac839f4..66293c89 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -306,6 +306,15 @@ dependencies = [
"crunchy",
]
+[[package]]
+name = "block-buffer"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
+dependencies = [
+ "generic-array",
+]
+
[[package]]
name = "boomphf"
version = "0.5.9"
@@ -538,6 +547,15 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+[[package]]
+name = "cpufeatures"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "crc32fast"
version = "1.3.2"
@@ -634,6 +652,16 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+[[package]]
+name = "crypto-mac"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
+dependencies = [
+ "generic-array",
+ "subtle",
+]
+
[[package]]
name = "cssparser"
version = "0.27.2"
@@ -736,6 +764,8 @@ dependencies = [
"regex",
"reqwest",
"rocksdb",
+ "rusoto_core",
+ "rusoto_s3",
"rust-stemmers",
"serde",
"serde_json",
@@ -747,6 +777,7 @@ dependencies = [
"tower-http",
"tracing",
"tracing-subscriber",
+ "urlencoding",
"uuid",
"whatlang",
]
@@ -779,6 +810,15 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
+[[package]]
+name = "digest"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
+dependencies = [
+ "generic-array",
+]
+
[[package]]
name = "dirs-next"
version = "2.0.0"
@@ -929,7 +969,7 @@ dependencies = [
"futures-sink",
"nanorand",
"pin-project",
- "spin",
+ "spin 0.9.4",
]
[[package]]
@@ -1103,6 +1143,16 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "generic-array"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
[[package]]
name = "getrandom"
version = "0.1.16"
@@ -1201,6 +1251,22 @@ dependencies = [
"libc",
]
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "hmac"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
+dependencies = [
+ "crypto-mac",
+ "digest",
+]
+
[[package]]
name = "html-escape"
version = "0.2.11"
@@ -1300,6 +1366,21 @@ dependencies = [
"want",
]
+[[package]]
+name = "hyper-rustls"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac"
+dependencies = [
+ "http",
+ "hyper",
+ "log",
+ "rustls",
+ "rustls-native-certs",
+ "tokio",
+ "tokio-rustls",
+]
+
[[package]]
name = "hyper-tls"
version = "0.5.0"
@@ -1668,6 +1749,17 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
+[[package]]
+name = "md-5"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"
+dependencies = [
+ "block-buffer",
+ "digest",
+ "opaque-debug",
+]
+
[[package]]
name = "md5"
version = "0.7.0"
@@ -1901,6 +1993,12 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
[[package]]
name = "openssl"
version = "0.10.40"
@@ -2461,6 +2559,21 @@ dependencies = [
"winreg",
]
+[[package]]
+name = "ring"
+version = "0.16.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin 0.5.2",
+ "untrusted",
+ "web-sys",
+ "winapi",
+]
+
[[package]]
name = "rocksdb"
version = "0.19.0"
@@ -2471,6 +2584,88 @@ dependencies = [
"librocksdb-sys",
]
+[[package]]
+name = "rusoto_core"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"
+dependencies = [
+ "async-trait",
+ "base64",
+ "bytes",
+ "crc32fast",
+ "futures",
+ "http",
+ "hyper",
+ "hyper-rustls",
+ "lazy_static",
+ "log",
+ "rusoto_credential",
+ "rusoto_signature",
+ "rustc_version",
+ "serde",
+ "serde_json",
+ "tokio",
+ "xml-rs",
+]
+
+[[package]]
+name = "rusoto_credential"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"
+dependencies = [
+ "async-trait",
+ "chrono",
+ "dirs-next",
+ "futures",
+ "hyper",
+ "serde",
+ "serde_json",
+ "shlex",
+ "tokio",
+ "zeroize",
+]
+
+[[package]]
+name = "rusoto_s3"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures",
+ "rusoto_core",
+ "xml-rs",
+]
+
+[[package]]
+name = "rusoto_signature"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
+dependencies = [
+ "base64",
+ "bytes",
+ "chrono",
+ "digest",
+ "futures",
+ "hex",
+ "hmac",
+ "http",
+ "hyper",
+ "log",
+ "md-5",
+ "percent-encoding",
+ "pin-project-lite",
+ "rusoto_credential",
+ "rustc_version",
+ "serde",
+ "sha2",
+ "tokio",
+]
+
[[package]]
name = "rust-stemmers"
version = "1.2.0"
@@ -2496,6 +2691,39 @@ dependencies = [
"semver",
]
+[[package]]
+name = "rustls"
+version = "0.20.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c"
+dependencies = [
+ "log",
+ "ring",
+ "sct",
+ "webpki",
+]
+
+[[package]]
+name = "rustls-native-certs"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"
+dependencies = [
+ "openssl-probe",
+ "rustls-pemfile",
+ "schannel",
+ "security-framework",
+]
+
+[[package]]
+name = "rustls-pemfile"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55"
+dependencies = [
+ "base64",
+]
+
[[package]]
name = "rustversion"
version = "1.0.6"
@@ -2545,6 +2773,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+[[package]]
+name = "sct"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
[[package]]
name = "security-framework"
version = "2.6.1"
@@ -2657,6 +2895,19 @@ dependencies = [
"stable_deref_trait",
]
+[[package]]
+name = "sha2"
+version = "0.9.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
+dependencies = [
+ "block-buffer",
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+ "opaque-debug",
+]
+
[[package]]
name = "sharded-slab"
version = "0.1.4"
@@ -2709,6 +2960,12 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
[[package]]
name = "spin"
version = "0.9.4"
@@ -2756,6 +3013,12 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+[[package]]
+name = "subtle"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+
[[package]]
name = "syn"
version = "1.0.96"
@@ -3082,6 +3345,17 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "tokio-rustls"
+version = "0.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
+dependencies = [
+ "rustls",
+ "tokio",
+ "webpki",
+]
+
[[package]]
name = "tokio-util"
version = "0.6.10"
@@ -3242,6 +3516,12 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
+[[package]]
+name = "typenum"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+
[[package]]
name = "unicase"
version = "2.6.0"
@@ -3284,6 +3564,12 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
[[package]]
name = "url"
version = "2.2.2"
@@ -3296,6 +3582,12 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "urlencoding"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9"
+
[[package]]
name = "utf-8"
version = "0.7.6"
@@ -3457,6 +3749,16 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "webpki"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
[[package]]
name = "weezl"
version = "0.1.7"
@@ -3565,6 +3867,18 @@ dependencies = [
"rand_core 0.6.3",
]
+[[package]]
+name = "xml-rs"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
+
+[[package]]
+name = "zeroize"
+version = "1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"
+
[[package]]
name = "zstd-sys"
version = "2.0.1+zstd.1.5.2"
diff --git a/Cargo.toml b/Cargo.toml
index 673e0e3b..3fd7c9ed 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,6 +7,9 @@ authors = ['Cuely ApS. <[email protected]>']
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+dev = ["rusoto_s3", "rusoto_core"]
+
[[bin]]
name = "cuely"
@@ -60,6 +63,9 @@ base64 = "0.13.0"
async-trait = "0.1.57"
boomphf = { version = "0.5.9", features = ["serde"] }
axum-macros = "0.2.3"
+urlencoding = "2.1.2"
+rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false, optional = true }
+rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"], optional = true }
[dev-dependencies]
criterion = "0.3.6"
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index 07e46301..dc15744e 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -112,7 +112,7 @@ import { askama } from "../askama";
</span>
))}
<span class="[&:nth-child(2)]:before:content-['—']">
- {askama`item.snippet $ {{lorem.lines}}`}
+ {askama`item.snippet|safe $ {{lorem.lines}}`}
</span>
</div>
</div>
diff --git a/justfile b/justfile
index 2d323990..b5b75f01 100644
--- a/justfile
+++ b/justfile
@@ -1,17 +1,7 @@
-@unpack-data:
- rm -rf data
- tar -zxvf data.tar.gz
-
@profile-indexer:
sudo rm -rf data/index
cargo flamegraph --root -- indexer local configs/indexer/profile.toml
-@worker:
- cargo run --release -- webgraph worker configs/webgraph/worker.toml
-
-@master:
- cargo run --release -- webgraph master configs/webgraph/master.toml
-
@webgraph:
rm -rf data/webgraph
cargo run --release -- webgraph local configs/webgraph/local.toml
@@ -30,17 +20,10 @@
@astro:
cd frontend; npm run dev
-@local:
- cd frontend; npm install
- rm -rf data/index/
- cargo run --release -- indexer local configs/indexer/local.toml
- mv data/index/CC-MAIN-*/* data/index/
- just frontend
+@configure:
+ cd frontend; npm install; npm run build
+ cargo run --release --all-features -- configure
@entity:
rm -rf data/entity
- cargo run --release -- indexer entity data/enwiki-20220801-pages-articles-multistream.xml.bz2 data/entity
-
-@pack-data:
- rm -f data.tar.gz
- tar --exclude="data/enwiki*" --exclude="data/warc_files" --exclude="data/webgraph" -zcvf data.tar.gz data
+ cargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entity
diff --git a/src/entrypoint/autosuggest_scrape.rs b/src/entrypoint/autosuggest_scrape.rs
new file mode 100644
index 00000000..294cd47b
--- /dev/null
+++ b/src/entrypoint/autosuggest_scrape.rs
@@ -0,0 +1,184 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ collections::{HashSet, VecDeque},
+ path::Path,
+ str::FromStr,
+};
+
+use csv::Writer;
+use indicatif::{ProgressBar, ProgressStyle};
+
+use crate::Result;
+
+fn suggestions(query: &str, gl: &str) -> Result<Vec<String>> {
+ let url = format!(
+ "https://www.google.com/complete/search?q={}&gl={}&client=gws-wiz&xssi=t",
+ urlencoding::encode(query),
+ gl
+ );
+
+ let client = reqwest::blocking::Client::new();
+ let builder = client.get(url);
+
+ let input = builder.send()?.text()?;
+
+ let mut input = input.split('\n');
+ input.next();
+ let input = input.next().expect("None option");
+
+ let output: serde_json::Value = serde_json::from_str(input)?;
+ let mut suggestions = Vec::new();
+
+ if let serde_json::Value::Array(arr) = output {
+ if let serde_json::Value::Array(arr) = arr[0].clone() {
+ for result in arr {
+ if let serde_json::Value::Array(result) = result {
+ if let serde_json::Value::String(result) = result[0].clone() {
+ let result = result.replace("<b>", "").replace("</b>", "");
+ suggestions.push(result);
+ }
+ }
+ }
+ }
+ }
+
+ Ok(suggestions)
+}
+
+pub enum Gl {
+ Us,
+}
+
+impl ToString for Gl {
+ fn to_string(&self) -> String {
+ match self {
+ Gl::Us => "us".to_string(),
+ }
+ }
+}
+
+impl FromStr for Gl {
+ type Err = crate::Error;
+
+ fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+ match s.to_ascii_lowercase().as_str() {
+ "us" => Ok(Gl::Us),
+ _ => Err(crate::Error::UnknownCLIOption),
+ }
+ }
+}
+
+fn save_queries<P: AsRef<Path>>(queries: &HashSet<String>, path: P) -> Result<()> {
+ let mut wtr = Writer::from_path(&path)?;
+
+ let mut queries: Vec<_> = queries.iter().collect();
+ queries.sort();
+
+ for query in queries {
+ wtr.write_record([query])?;
+ }
+
+ wtr.flush()?;
+
+ Ok(())
+}
+
+pub fn run<P: AsRef<Path>>(
+ queries_to_scrape: usize,
+ gl: Gl,
+ ms_sleep_between_req: u64,
+ output_dir: P,
+) -> Result<()> {
+ let mut queries = HashSet::new();
+ let mut queue = VecDeque::new();
+
+ let pb = ProgressBar::new(queries_to_scrape as u64);
+ pb.set_style(
+ ProgressStyle::default_bar()
+ .template("{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})")
+ .progress_chars("#>-"),
+ );
+
+ for c in 'a'..='z' {
+ queue.push_back(c.to_string());
+ }
+
+ let path = output_dir
+ .as_ref()
+ .join(format!("queries_{:}.csv", gl.to_string().as_str()));
+
+ let mut queries_since_last_save = 0;
+
+ while let Some(query) = queue.pop_front() {
+ let res = suggestions(&query, gl.to_string().as_str());
+
+ if res.is_err() {
+ continue;
+ }
+
+ let res = res.unwrap();
+ let mut new_queries = 0;
+
+ for next_query in res {
+ if queries.contains(&next_query) {
+ continue;
+ }
+
+ let mut new = Vec::new();
+ for c in next_query.chars() {
+ let q = new.clone().into_iter().collect();
+ new.push(c);
+
+ if !queries.contains(&q) {
+ queue.push_back(q);
+ }
+ }
+
+ for q in next_query.split_whitespace() {
+ if !queries.contains(q) {
+ queries.insert(q.to_string());
+ queue.push_back(q.to_string());
+ }
+ }
+
+ new_queries += 1;
+ queries.insert(next_query);
+ pb.tick();
+ pb.set_position(queries.len() as u64);
+ }
+
+ if queries.len() >= queries_to_scrape {
+ break;
+ }
+
+ queries_since_last_save += new_queries;
+
+ if queries_since_last_save > 1_000 {
+ save_queries(&queries, &path)?;
+ queries_since_last_save = 0;
+ }
+
+ std::thread::sleep(std::time::Duration::from_millis(ms_sleep_between_req));
+ }
+
+ pb.finish();
+
+ save_queries(&queries, &path)?;
+
+ Ok(())
+}
diff --git a/src/entrypoint/configure.rs b/src/entrypoint/configure.rs
new file mode 100644
index 00000000..95556fea
--- /dev/null
+++ b/src/entrypoint/configure.rs
@@ -0,0 +1,186 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rusoto_core::credential::StaticProvider;
+use rusoto_core::HttpClient;
+use rusoto_s3::{GetObjectRequest, S3Client, S3};
+use tokio::fs::File;
+use tokio::io;
+use tracing::{debug, info};
+
+use crate::entrypoint::indexer;
+use crate::Result;
+use std::fs::{self};
+use std::path::Path;
+
+use super::{webgraph, Centrality, EntityIndexer};
+
+const DATA_PATH: &str = "data";
+const BUCKET_NAME: &str = "s3.cuely.io";
+
+fn download_files() {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async {
+ let creds = StaticProvider::new("".to_string(), "".to_string(), None, None);
+ let client = S3Client::new_with(
+ HttpClient::new().unwrap(),
+ creds,
+ rusoto_core::Region::EuCentral1,
+ );
+
+ for name in [
+ "enwiki_subset.xml.bz2",
+ "queries_us.csv",
+ "sample.warc.gz",
+ "bangs.json",
+ ] {
+ info!("Downloading {}", name);
+ let mut object = client
+ .get_object(GetObjectRequest {
+ bucket: BUCKET_NAME.into(),
+ key: name.into(),
+ ..Default::default()
+ })
+ .await
+ .unwrap();
+
+ let body = object.body.take().expect("The object has no body");
+
+ let mut body = body.into_async_read();
+ let mut file = File::create(Path::new(DATA_PATH).join(name)).await.unwrap();
+ io::copy(&mut body, &mut file).await.unwrap();
+ }
+ });
+}
+
+fn create_webgraph() -> Result<()> {
+ debug!("Creating webgraph");
+ let out_path_tmp = Path::new(DATA_PATH).join("webgraph_tmp");
+ let out_path = Path::new(DATA_PATH).join("webgraph");
+
+ if out_path_tmp.exists() {
+ std::fs::remove_dir_all(&out_path_tmp)?;
+ }
+ if out_path.exists() {
+ std::fs::remove_dir_all(&out_path)?;
+ }
+
+ let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");
+
+ let job = webgraph::Job {
+ config: webgraph::JobConfig::Local(crate::LocalConfig {
+ folder: ".".to_string(),
+ names: vec![warc_path.to_str().unwrap().to_string()],
+ }),
+ warc_paths: vec![warc_path.to_str().unwrap().to_string()],
+ graph_base_path: out_path_tmp.to_str().unwrap().to_string(),
+ };
+
+ let graph = webgraph::process_job(&job);
+ std::fs::rename(graph.path, out_path)?;
+ std::fs::remove_dir_all(&out_path_tmp)?;
+
+ Ok(())
+}
+
+fn calculate_centrality() {
+ debug!("Calculating centrality");
+ let webgraph_path = Path::new(DATA_PATH).join("webgraph");
+ let out_path = Path::new(DATA_PATH).join("centrality");
+ Centrality::run(webgraph_path, out_path);
+}
+
+fn create_inverted_index() -> Result<()> {
+ debug!("Creating inverted index");
+ let out_path = Path::new(DATA_PATH).join("index");
+ let out_path_tmp = Path::new(DATA_PATH).join("index_tmp");
+
+ if out_path_tmp.exists() {
+ std::fs::remove_dir_all(&out_path_tmp)?;
+ }
+ if out_path.exists() {
+ std::fs::remove_dir_all(&out_path)?;
+ }
+
+ let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");
+
+ let job = indexer::Job {
+ source_config: indexer::JobConfig::Local(crate::LocalConfig {
+ folder: ".".to_string(),
+ names: vec![warc_path.to_str().unwrap().to_string()],
+ }),
+ download_images: false,
+ warc_paths: vec![warc_path.to_str().unwrap().to_string()],
+ base_path: out_path_tmp.to_str().unwrap().to_string(),
+ host_centrality_threshold: None,
+ max_num_segments: 1,
+ };
+
+ let webgraph_path = Path::new(DATA_PATH).join("webgraph");
+ let centrality_path = Path::new(DATA_PATH).join("centrality");
+
+ let worker = indexer::IndexingWorker::new(
+ centrality_path.to_str().unwrap().to_string(),
+ Some(webgraph_path.to_str().unwrap().to_string()),
+ );
+
+ let index = indexer::process_job(&job, &worker);
+ std::fs::rename(index.path, out_path)?;
+ std::fs::remove_dir_all(&out_path_tmp)?;
+
+ Ok(())
+}
+
+fn create_entity_index() -> Result<()> {
+ let out_path = Path::new(DATA_PATH).join("entity");
+ if out_path.exists() {
+ std::fs::remove_dir_all(&out_path)?;
+ }
+
+ let wiki_path = Path::new(DATA_PATH).join("enwiki_subset.xml.bz2");
+
+ EntityIndexer::run(
+ wiki_path.to_str().unwrap().to_string(),
+ out_path.to_str().unwrap().to_string(),
+ )?;
+
+ Ok(())
+}
+
+fn index_files() -> Result<()> {
+ create_webgraph()?;
+ calculate_centrality();
+ create_inverted_index()?;
+ create_entity_index()?;
+
+ Ok(())
+}
+
+pub fn run() -> Result<()> {
+ let p = Path::new(DATA_PATH);
+
+ if !p.exists() {
+ fs::create_dir_all(p)?;
+ }
+
+ download_files();
+ index_files()?;
+
+ Ok(())
+}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index af1229f8..73ec6f04 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -38,28 +38,28 @@ use crate::{
pub struct Indexer {}
#[derive(Debug, Serialize, Deserialize, Clone)]
-enum JobConfig {
+pub enum JobConfig {
Http(HttpConfig),
Local(LocalConfig),
}
#[derive(Debug, Serialize, Deserialize)]
-struct Job {
- source_config: JobConfig,
- download_images: bool,
- warc_paths: Vec<String>,
- base_path: String,
- host_centrality_threshold: Option<f64>,
- max_num_segments: u32,
+pub struct Job {
+ pub source_config: JobConfig,
+ pub download_images: bool,
+ pub warc_paths: Vec<String>,
+ pub base_path: String,
+ pub host_centrality_threshold: Option<f64>,
+ pub max_num_segments: u32,
}
-struct IndexingWorker {
+pub struct IndexingWorker {
centrality_store: CentralityStore,
webgraph: Option<Webgraph>,
}
impl IndexingWorker {
- fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {
+ pub fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {
Self {
centrality_store: CentralityStore::open(centrality_store_path),
webgraph: webgraph_path.map(|path| {
@@ -202,7 +202,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
index
}
-fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
+pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index 9ab62d1f..549241f3 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -13,7 +13,10 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub mod autosuggest_scrape;
mod centrality;
+#[cfg(feature = "dev")]
+pub mod configure;
mod entity;
pub mod frontend;
pub mod indexer;
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 5ae7f73f..0494397f 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -34,16 +34,16 @@ use tracing::{info, trace};
struct GraphPointer(String);
#[derive(Debug, Serialize, Deserialize, Clone)]
-enum JobConfig {
+pub enum JobConfig {
Http(HttpConfig),
Local(LocalConfig),
}
#[derive(Debug, Serialize, Deserialize)]
-struct Job {
- config: JobConfig,
- warc_paths: Vec<String>,
- graph_base_path: String,
+pub struct Job {
+ pub config: JobConfig,
+ pub warc_paths: Vec<String>,
+ pub graph_base_path: String,
}
fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
@@ -105,7 +105,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
graph
}
-fn process_job(job: &Job) -> webgraph::Webgraph {
+pub fn process_job(job: &Job) -> webgraph::Webgraph {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
diff --git a/src/image_downloader.rs b/src/image_downloader.rs
index 344ccfa0..97e56fd5 100644
--- a/src/image_downloader.rs
+++ b/src/image_downloader.rs
@@ -44,7 +44,7 @@ where
continue;
}
- for duration in ExponentialBackoff::from_millis(10).take(5) {
+ for duration in ExponentialBackoff::from_millis(10).take(4) {
if let Some(image) = url
.download_bytes(self.timeout.unwrap_or_else(|| Duration::from_secs(20)))
.await
@@ -123,17 +123,25 @@ where
}
async fn download_pending_images_async(&mut self, store: &mut impl ImageStore<K>) {
- let results = futures::stream::iter(
+ let mut stream = futures::stream::iter(
self.image_download_jobs
.drain()
.map(|job| async move { job.download().await }),
)
- .buffer_unordered(20)
- .collect::<Vec<Option<DownloadedImage<K>>>>()
- .await;
+ .buffer_unordered(20);
- for result in results.into_iter().flatten() {
- store.insert(result.key, result.image);
+ let mut inserts_since_flush = 0;
+
+ while let Some(result) = stream.next().await {
+ if let Some(result) = result {
+ store.insert(result.key, result.image);
+ inserts_since_flush += 1;
+ }
+
+ if inserts_since_flush > 1_000 {
+ store.flush();
+ inserts_since_flush = 0;
+ }
}
store.flush();
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 8a1adbe0..8b79f4a0 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -34,6 +34,7 @@ use crate::webpage::region::Region;
use crate::webpage::{StoredPrimaryImage, Webpage};
use crate::Result;
use crate::{schema::create_schema, tokenizer::Tokenizer};
+use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::sync::{Arc, Weak};
@@ -306,50 +307,60 @@ impl InvertedIndex {
}
pub fn merge(mut self, mut other: InvertedIndex) -> Self {
- other.commit().expect("failed to commit index");
- self.commit().expect("failed to commit index");
+ let path = self.path.clone();
- let other_meta = other
- .tantivy_index
- .load_metas()
- .expect("failed to laod tantivy metadata for index");
+ {
+ other.commit().expect("failed to commit index");
+ self.commit().expect("failed to commit index");
- let mut meta = self
- .tantivy_index
- .load_metas()
- .expect("failed to laod tantivy metadata for index");
+ let other_meta = other
+ .tantivy_index
+ .load_metas()
+ .expect("failed to load tantivy metadata for index");
- let x = other.path.clone();
- let other_path = Path::new(x.as_str());
- other.writer.wait_merging_threads().unwrap();
+ let mut meta = self
+ .tantivy_index
+ .load_metas()
+ .expect("failed to load tantivy metadata for index");
- let path = self.path.clone();
- let self_path = Path::new(path.as_str());
- self.writer.wait_merging_threads().unwrap();
-
- for segment in other_meta.segments {
- // TODO: handle case where current index has segment with same name
- for file in segment.list_files() {
- let p = other_path.join(&file);
- if p.exists() {
- fs::rename(p, self_path.join(&file)).unwrap();
+ let x = other.path.clone();
+ let other_path = Path::new(x.as_str());
+ other.writer.wait_merging_threads().unwrap();
+
+ let path = self.path.clone();
+ let self_path = Path::new(path.as_str());
+ self.writer.wait_merging_threads().unwrap();
+
+ let ids: HashSet<_> = meta.segments.iter().map(|segment| segment.id()).collect();
+
+ for segment in other_meta.segments {
+ if ids.contains(&segment.id()) {
+ continue;
}
+
+ // TODO: handle case where current index has segment with same name
+ for file in segment.list_files() {
+ let p = other_path.join(&file);
+ if p.exists() {
+ fs::rename(p, self_path.join(&file)).unwrap();
+ }
+ }
+ meta.segments.push(segment);
}
- meta.segments.push(segment);
- }
- meta.segments
- .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));
+ meta.segments
+ .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));
- fs::remove_dir_all(other_path).unwrap();
+ fs::remove_dir_all(other_path).unwrap();
- let self_path = Path::new(&path);
+ let self_path = Path::new(&path);
- std::fs::write(
- self_path.join("meta.json"),
- serde_json::to_string_pretty(&meta).unwrap(),
- )
- .unwrap();
+ std::fs::write(
+ self_path.join("meta.json"),
+ serde_json::to_string_pretty(&meta).unwrap(),
+ )
+ .unwrap();
+ }
Self::open(path).expect("failed to open index")
}
diff --git a/src/lib.rs b/src/lib.rs
index a89d57d0..0eca3671 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -132,8 +132,8 @@ impl WarcSource {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct LocalConfig {
- folder: String,
- names: Vec<String>,
+ pub folder: String,
+ pub names: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -222,6 +222,12 @@ pub enum Error {
#[error("Could not open inverted-index directory")]
Directory(#[from] tantivy::directory::error::OpenDirectoryError),
+
+ #[error("Could not convert to/from JSON")]
+ Json(#[from] serde_json::Error),
+
+ #[error("Unknown CLI option")]
+ UnknownCLIOption,
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/main.rs b/src/main.rs
index aa0dfaab..f780f45d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,6 +15,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
use clap::{Parser, Subcommand};
+use cuely::entrypoint::autosuggest_scrape::{self, Gl};
+#[cfg(feature = "dev")]
+use cuely::entrypoint::configure;
use cuely::entrypoint::indexer::IndexPointer;
use cuely::entrypoint::{self, frontend, search_server};
use cuely::{FrontendConfig, SearchServerConfig};
@@ -52,6 +55,14 @@ enum Commands {
Frontend {
config_path: String,
},
+ AutosuggestScrape {
+ num_queries: usize,
+ gl: Gl,
+ ms_sleep_between_req: u64,
+ output_dir: String,
+ },
+ #[cfg(feature = "dev")]
+ Configure,
}
#[derive(Subcommand)]
@@ -163,6 +174,16 @@ fn main() -> Result<()> {
.build()?
.block_on(search_server::run(config))?
}
+ Commands::AutosuggestScrape {
+ num_queries: queries_to_scrape,
+ gl,
+ ms_sleep_between_req,
+ output_dir,
+ } => {
+ autosuggest_scrape::run(queries_to_scrape, gl, ms_sleep_between_req, output_dir)?;
+ }
+ #[cfg(feature = "dev")]
+ Commands::Configure => configure::run()?,
}
Ok(())
diff --git a/src/warc.rs b/src/warc.rs
index eecb7b01..a709178c 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -25,7 +25,7 @@ use std::time::Duration;
use flate2::read::MultiGzDecoder;
use futures::StreamExt;
use tokio::time::sleep;
-use tracing::debug;
+use tracing::{debug, trace};
pub(crate) struct WarcFile {
bytes: Vec<u8>,
@@ -108,6 +108,8 @@ impl WarcFile {
if res.is_ok() {
return Ok(());
+ } else {
+ trace!("Error {:?}", res);
}
debug!("warc download failed: {:?}", res.err().unwrap());
diff --git a/src/webgraph/centrality/betweenness.rs b/src/webgraph/centrality/betweenness.rs
index b43b1243..f02622bf 100644
--- a/src/webgraph/centrality/betweenness.rs
+++ b/src/webgraph/centrality/betweenness.rs
@@ -127,7 +127,7 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<N
}
if let Some(pb) = &pb {
- pb.finish();
+ pb.finish_and_clear();
}
let n = n as f64;