web search done right
crawler
Mikkel Denkeraacc1e3dc527
diff --git a/.cargo/config.toml b/.cargo/config.tomlindex d47df8ab..ff5fc321 100644--- a/.cargo/config.toml+++ b/.cargo/config.toml@@ -3,4 +3,5 @@ rustflags = ["-C", "target-cpu=native"][env]LIBTORCH = {value = ".venv/lib/python3.11/site-packages/torch", force=true, relative=true}-DYLD_LIBRARY_PATH = {value = ".venv/lib/python3.11/site-packages/torch/lib", force=true, relative=true}\ No newline at end of file+DYLD_LIBRARY_PATH = {value = ".venv/lib/python3.11/site-packages/torch/lib", force=true, relative=true}+LIBSQLITE3_FLAGS = { value = "-DSQLITE_ENABLE_MATH_FUNCTIONS", force=true }\ No newline at end of filediff --git a/Cargo.lock b/Cargo.lockindex 756eed4d..a1190990 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -66,6 +66,17 @@ dependencies = ["version_check",]+[[package]]+name = "ahash"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"+dependencies = [+ "cfg-if",+ "once_cell",+ "version_check",+]+[[package]]name = "aho-corasick"version = "0.7.20"@@ -84,6 +95,12 @@ dependencies = ["memchr",]+[[package]]+name = "allocator-api2"+version = "0.2.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4f263788a35611fba42eb41ff811c5d0360c58b97402570312a350736e2542e"+[[package]]name = "android_system_properties"version = "0.1.5"@@ -257,6 +274,20 @@ dependencies = ["syn 2.0.15",]+[[package]]+name = "attohttpc"+version = "0.22.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7"+dependencies = [+ "http",+ "log",+ "native-tls",+ "serde",+ "serde_json",+ "url",+]+[[package]]name = "atty"version = "0.2.14"@@ -274,6 +305,32 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"+[[package]]+name = "aws-creds"+version = "0.34.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5"+dependencies = [+ "attohttpc",+ "dirs",+ "log",+ "quick-xml 0.26.0",+ "rust-ini",+ "serde",+ "thiserror",+ "time 0.3.20",+ "url",+]++[[package]]+name = "aws-region"+version = "0.25.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba"+dependencies = [+ "thiserror",+]+[[package]]name = "axum"version = "0.6.16"@@ -282,7 +339,7 @@ checksum = "113713495a32dd0ab52baf5c10044725aa3aec00b31beda84218e469029b72a3"dependencies = ["async-trait","axum-core",- "bitflags",+ "bitflags 1.3.2","bytes","futures-util","http",@@ -404,7 +461,7 @@ version = "0.64.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"dependencies = [- "bitflags",+ "bitflags 1.3.2","cexpr","clang-sys","lazy_static",@@ -445,6 +502,12 @@ version = "1.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"+[[package]]+name = "bitflags"+version = "2.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded"+[[package]]name = "bitpacking"version = "0.8.4"@@ -668,7 +731,7 @@ version = "2.34.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"dependencies = [- "bitflags",+ "bitflags 1.3.2","textwrap 0.11.0","unicode-width",]@@ -680,7 +743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"dependencies = ["atty",- "bitflags",+ "bitflags 1.3.2","clap_derive 3.2.18","clap_lex 0.2.4","indexmap",@@ -709,7 +772,7 @@ checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"dependencies = ["anstream","anstyle",- "bitflags",+ "bitflags 1.3.2","clap_lex 0.4.1","strsim",]@@ -1083,7 +1146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"dependencies = ["cfg-if",- "hashbrown",+ "hashbrown 0.12.3","lock_api","once_cell","parking_lot_core",@@ -1191,6 +1254,12 @@ dependencies = ["winapi",]+[[package]]+name = "dlv-list"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"+[[package]]name = "downcast-rs"version = "1.2.0"@@ -1310,6 +1379,18 @@ dependencies = ["rand 0.8.5",]+[[package]]+name = "fallible-iterator"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"++[[package]]+name = "fallible-streaming-iterator"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"+[[package]]name = "fastdivide"version = "0.4.0"@@ -1672,7 +1753,26 @@ version = "0.12.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"dependencies = [- "ahash",+ "ahash 0.7.6",+]++[[package]]+name = "hashbrown"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"+dependencies = [+ "ahash 0.8.3",+ "allocator-api2",+]++[[package]]+name = "hashlink"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f"+dependencies = [+ "hashbrown 0.14.0",][[package]]@@ -1714,6 +1814,12 @@ version = "0.3.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"+[[package]]+name = "hex"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"+[[package]]name = "histogram"version = "0.6.9"@@ -1907,7 +2013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"dependencies = ["autocfg",- "hashbrown",+ "hashbrown 0.12.3",][[package]]@@ -2150,6 +2256,17 @@ dependencies = ["zstd-sys",]+[[package]]+name = "libsqlite3-sys"+version = "0.26.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"+dependencies = [+ "cc",+ "pkg-config",+ "vcpkg",+]+[[package]]name = "libz-sys"version = "1.1.8"@@ -2238,7 +2355,7 @@ version = "0.7.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"dependencies = [- "hashbrown",+ "hashbrown 0.12.3",][[package]]@@ -2328,6 +2445,17 @@ dependencies = ["rawpointer",]+[[package]]+name = "maybe-async"+version = "0.2.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]+[[package]]name = "md5"version = "0.7.0"@@ -2400,6 +2528,15 @@ version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2687e6cf9c00f48e9284cf9fd15f2ef341d03cc7743abf9df4c5f07fdee50b18"+[[package]]+name = "minidom"+version = "0.15.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278"+dependencies = [+ "rxml",+]+[[package]]name = "minimal-lexical"version = "0.2.1"@@ -2653,7 +2790,7 @@ version = "6.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"dependencies = [- "bitflags",+ "bitflags 1.3.2","libc","once_cell","onig_sys",@@ -2687,7 +2824,7 @@ version = "0.10.51"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "97ea2d98598bf9ada7ea6ee8a30fb74f9156b63bbe495d64ec2b87c269d2dda3"dependencies = [- "bitflags",+ "bitflags 1.3.2","cfg-if","foreign-types","libc",@@ -2738,6 +2875,16 @@ dependencies = ["thiserror",]+[[package]]+name = "ordered-multimap"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"+dependencies = [+ "dlv-list",+ "hashbrown 0.12.3",+]+[[package]]name = "os_str_bytes"version = "6.5.0"@@ -2990,7 +3137,7 @@ version = "0.17.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa"dependencies = [- "bitflags",+ "bitflags 1.3.2","crc32fast","fdeflate","flate2",@@ -3124,6 +3271,16 @@ dependencies = ["memchr",]+[[package]]+name = "quick-xml"+version = "0.26.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"+dependencies = [+ "memchr",+ "serde",+]+[[package]]name = "quote"version = "1.0.26"@@ -3265,7 +3422,7 @@ version = "0.2.16"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"dependencies = [- "bitflags",+ "bitflags 1.3.2",][[package]]@@ -3274,7 +3431,7 @@ version = "0.3.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"dependencies = [- "bitflags",+ "bitflags 1.3.2",][[package]]@@ -3391,7 +3548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "21499ed91807f07ae081880aabb2ccc0235e9d88011867d984525e9a4c3cfa3e"dependencies = ["bytecheck",- "hashbrown",+ "hashbrown 0.12.3","ptr_meta","rend","rkyv_derive",@@ -3409,6 +3566,12 @@ dependencies = ["syn 1.0.109",]+[[package]]+name = "robotstxt-with-cache"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "413592c1c83b4657fd53561be2e58dc4d739caa90c954d7799b827312f6a3fe8"+[[package]]name = "rocksdb"version = "0.19.0"@@ -3419,6 +3582,63 @@ dependencies = ["librocksdb-sys",]+[[package]]+name = "rusqlite"+version = "0.29.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"+dependencies = [+ "bitflags 2.3.2",+ "fallible-iterator",+ "fallible-streaming-iterator",+ "hashlink",+ "libsqlite3-sys",+ "smallvec",+]++[[package]]+name = "rust-ini"+version = "0.18.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"+dependencies = [+ "cfg-if",+ "ordered-multimap",+]++[[package]]+name = "rust-s3"+version = "0.33.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027"+dependencies = [+ "async-trait",+ "aws-creds",+ "aws-region",+ "base64 0.13.1",+ "bytes",+ "cfg-if",+ "futures",+ "hex",+ "hmac",+ "http",+ "log",+ "maybe-async",+ "md5",+ "minidom",+ "percent-encoding",+ "quick-xml 0.26.0",+ "reqwest",+ "serde",+ "serde_derive",+ "sha2",+ "thiserror",+ "time 0.3.20",+ "tokio",+ "tokio-stream",+ "url",+]+[[package]]name = "rust-stemmers"version = "1.2.0"@@ -3450,7 +3670,7 @@ version = "0.37.14"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f"dependencies = [- "bitflags",+ "bitflags 1.3.2","errno","io-lifetimes","libc",@@ -3464,6 +3684,23 @@ version = "1.0.12"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"+[[package]]+name = "rxml"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7"+dependencies = [+ "bytes",+ "rxml_validation",+ "smartstring",+]++[[package]]+name = "rxml_validation"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530"+[[package]]name = "ryu"version = "1.0.13"@@ -3591,7 +3828,7 @@ version = "2.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"dependencies = [- "bitflags",+ "bitflags 1.3.2","core-foundation","core-foundation-sys","libc",@@ -3614,7 +3851,7 @@ version = "0.22.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"dependencies = [- "bitflags",+ "bitflags 1.3.2","cssparser","derive_more","fxhash",@@ -3785,6 +4022,17 @@ version = "1.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"+[[package]]+name = "smartstring"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"+dependencies = [+ "autocfg",+ "static_assertions",+ "version_check",+]+[[package]]name = "snap"version = "1.1.0"@@ -3885,14 +4133,17 @@ dependencies = ["optics","parse_wiki_text","publicsuffix",- "quick-xml",+ "quick-xml 0.23.1","rand 0.8.5","rayon","regex","reqwest","reqwest-eventsource","rkyv",+ "robotstxt-with-cache","rocksdb",+ "rusqlite",+ "rust-s3","rust-stemmers","safetensors","scylla",@@ -4465,7 +4716,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"dependencies = ["async-compression",- "bitflags",+ "bitflags 1.3.2","bytes","futures-core","futures-util",@@ -4862,7 +5113,7 @@ version = "0.16.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9c531a2dc4c462b833788be2c07eef4e621d0e9edbd55bf280cc164c1c1aa043"dependencies = [- "hashbrown",+ "hashbrown 0.12.3","once_cell",]diff --git a/Cargo.toml b/Cargo.tomlindex 7a502313..5f2286d0 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -85,4 +85,7 @@ torch-sys = "0.13.0"libc = "0.2.142"lz-str = "0.2.1"safetensors = "0.3.1"-publicsuffix = "2"\ No newline at end of file+publicsuffix = "2"+robotstxt-with-cache = "0.4.0"+rust-s3 = "0.33.0"+rusqlite = { version = "0.29.0", features = ["bundled"] }\ No newline at end of filediff --git a/configs/crawler/coordinator.toml b/configs/crawler/coordinator.tomlnew file mode 100644index 00000000..43018773--- /dev/null+++ b/configs/crawler/coordinator.toml@@ -0,0 +1,4 @@+crawldb_folder = "data/crawldb"+host = "0.0.0.0:8080"+num_urls_to_crawl = 3+seed_urls = ["https://weld.app"]\ No newline at end of filediff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.tomlnew file mode 100644index 00000000..8ec5f1bf--- /dev/null+++ b/configs/crawler/crawler.toml@@ -0,0 +1,12 @@+num_workers = 10+user_agent = "Mozilla/5.0 (compatible; Stractbot/0.1; +https://trystract.com/bot)"+politeness_factor = 1.0+timeout_seconds = 30+coordinator_host = "0.0.0.0:8080"++[s3]+bucket = "crawl"+folder = "test1"+access_key = "<access_key>"+secret_key = "<secret_key>"+endpoint = "https://s3.trystract.com"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex ef0f8691..92a9ecee 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -88,6 +88,9 @@ half = { workspace = true }aes-gcm = { workspace = true }reqwest-eventsource = { workspace = true }publicsuffix = { workspace = true }+robotstxt-with-cache = { workspace = true }+rust-s3 = { workspace = true }+rusqlite = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/core/src/api/crawler.rs b/core/src/api/crawler.rsnew file mode 100644index 00000000..f2dcb117--- /dev/null+++ b/core/src/api/crawler.rs@@ -0,0 +1,29 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::HtmlTemplate;+use askama::Template;+use axum::response::IntoResponse;++#[allow(clippy::unused_async)]+pub async fn info_route() -> impl IntoResponse {+ let template = CrawlerTemplate {};+ HtmlTemplate(template)+}++#[derive(Template)]+#[template(path = "crawler/index.html")]+struct CrawlerTemplate {}diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 56344b3a..5e7576c8 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -51,6 +51,7 @@ mod about;mod alice;mod autosuggest;mod chat;+mod crawler;mod explore;mod fact_check;pub mod improvement;@@ -177,6 +178,7 @@ pub async fn router(.route("/settings/sites", get(sites::route)).route("/settings/privacy", get(improvement::settings)).route("/privacy-and-happy-lawyers", get(privacy::route))+ .route("/bot", get(crawler::info_route)).route("/opensearch.xml", get(opensearch::route)).route("/improvement/click", post(improvement::click)).route("/improvement/store", post(improvement::store))diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsnew file mode 100644index 00000000..667bbfc4--- /dev/null+++ b/core/src/crawler/coordinator.rs@@ -0,0 +1,437 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use rusqlite::{+ types::{FromSql, FromSqlError, FromSqlResult, ValueRef},+ ToSql,+};++use crate::webpage::Url;++use super::{Domain, Job, JobResponse, Result, UrlResponse};+use std::{+ collections::HashSet,+ path::Path,+ sync::{+ atomic::{AtomicU64, Ordering},+ Arc,+ },+ time::Instant,+};++const DEFAULT_JOB_URLS: usize = 1000;++enum UrlStatus {+ Pending,+ Crawling,+ Failed,+ Done,+}++impl ToSql for UrlStatus {+ fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {+ match self {+ UrlStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(0),+ )),+ UrlStatus::Crawling => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(1),+ )),+ UrlStatus::Failed => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(2),+ )),+ UrlStatus::Done => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(3),+ )),+ }+ }+}++impl FromSql for UrlStatus {+ fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {+ match value {+ ValueRef::Integer(0) => Ok(UrlStatus::Pending),+ ValueRef::Integer(1) => Ok(UrlStatus::Crawling),+ ValueRef::Integer(2) => Ok(UrlStatus::Failed),+ ValueRef::Integer(3) => Ok(UrlStatus::Done),+ _ => Err(FromSqlError::InvalidType),+ }+ }+}++enum DomainStatus {+ Pending,+ CrawlInProgress,+}++impl ToSql for DomainStatus {+ fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {+ match self {+ DomainStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(0),+ )),+ DomainStatus::CrawlInProgress => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(1),+ )),+ }+ }+}++impl FromSql for DomainStatus {+ fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {+ match value {+ ValueRef::Integer(0) => Ok(DomainStatus::Pending),+ ValueRef::Integer(1) => Ok(DomainStatus::CrawlInProgress),+ _ => Err(FromSqlError::InvalidType),+ }+ }+}++pub struct CrawlCoordinator {+ conn: Arc<rusqlite::Connection>,+ num_crawled_urls: AtomicU64,+ num_urls_to_crawl: u64,+}++impl CrawlCoordinator {+ pub fn new<P: AsRef<Path>>(+ crawldb_folder: P,+ num_urls_to_crawl: u64,+ seed_urls: Vec<String>,+ ) -> Result<Self> {+ let conn = Arc::new(rusqlite::Connection::open(crawldb_folder)?);++ // create tables if not exists+ // there should be one table that contains all known URLs+ // when a URL is added to the table, it should be marked as `UrlStatus::Pending`+ // when a URL is being fetched, it should be marked as `UrlStatus::Crawling`+ // when a URL has been fetched, it should be marked as `UrlStatus::Done`.+ // Each URL should have a counter that counts the number of incoming links+ // from URLs on other domains.+ conn.execute(+ "CREATE TABLE IF NOT EXISTS url(+ url TEXT NOT NULL UNIQUE,+ domain TEXT NOT NULL,+ status INTEGER NOT NULL,+ error_code INTEGER,+ incoming_links INTEGER NOT NULL,+ PRIMARY KEY (url)+ );",+ [],+ )?;++ // index by url and status+ conn.execute(+ "CREATE INDEX IF NOT EXISTS url_url_status ON url (url, status);",+ [],+ )?;++ // there should be one table that contains all known domains+ // and whether a crawl is in progress for that domain.+ // It should also contain a copy of the maximum count of incoming links+ // for any URL in that domain.+ conn.execute(+ "CREATE TABLE IF NOT EXISTS domain (+ domain TEXT NOT NULL UNIQUE,+ max_incoming_links INTEGER NOT NULL,+ status INTEGER NOT NULL,+ PRIMARY KEY (domain)+ );",+ [],+ )?;++ // index by domain and status+ conn.execute(+ "CREATE INDEX IF NOT EXISTS domain_domain_status ON domain (domain, status);",+ [],+ )?;++ // store redirects+ conn.execute(+ "CREATE TABLE IF NOT EXISTS redirect (+ from_url INTEGER NOT NULL,+ to_url INTEGER NOT NULL,+ PRIMARY KEY (from_url, to_url)+ );",+ [],+ )?;++ // update performance stuff+ // WAL mode+ conn.pragma_update(None, "journal_mode", "'WAL'")?;++ // sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.+ conn.pragma_update(None, "synchronous", 0)?;++ // store temp tables in memory+ conn.pragma_update(None, "temp_store", 2)?;++ // set cache size to 512 MB+ conn.pragma_update(None, "cache_size", -512_000)?; // negative value means kilobytes (https://www.sqlite.org/pragma.html#pragma_cache_size)++ let domain_seeds = seed_urls+ .iter()+ .map(|url| Domain(Url::from(url.clone()).domain().to_string()))+ .collect::<HashSet<_>>();++ {+ let mut prepared_insert_url = conn.prepare(+ "INSERT INTO url (url, domain, status, incoming_links)+ VALUES (?1, ?2, ?3, ?4)+ ON CONFLICT (url) DO NOTHING;",+ )?;+ let mut prepared_insert_domain = conn.prepare(+ "INSERT INTO domain (domain, max_incoming_links, status)+ VALUES (?1, 0, ?2)+ ON CONFLICT (domain) DO NOTHING;",+ )?;++ // insert seed URLs+ for url in seed_urls {+ let url = Url::from(url);+ prepared_insert_url.execute((+ url.full(),+ url.domain().to_string(),+ UrlStatus::Pending,+ 0,+ ))?;+ prepared_insert_domain+ .execute((url.domain().to_string(), DomainStatus::Pending))?;+ }+ }++ let coord = Self {+ conn,+ num_urls_to_crawl,+ num_crawled_urls: AtomicU64::new(0),+ };++ coord.update_max_inlinks_domains(domain_seeds.iter())?;++ Ok(coord)+ }++ pub fn add_response(&self, response: &JobResponse) -> Result<()> {+ self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);+ // begin transaction+ let tx = self.conn.unchecked_transaction()?;++ {+ // insert discovered URLs if not already in database+ // update incoming link counts for discovered URLs+ // if count is above prev max for domain, update max+ let domain = Url::from(response.domain.0.clone());+ let mut prepared_diff_domain = tx.prepare(+ "INSERT INTO url (url, domain, status, incoming_links)+ VALUES (?1, ?2, ?3, 1)+ ON CONFLICT (url) DO UPDATE SET incoming_links = incoming_links + 1;",+ )?;+ let mut prepared_same_domain = tx.prepare(+ "INSERT INTO url (url, domain, status, incoming_links)+ VALUES (?1, ?2, ?3, 0)+ ON CONFLICT (url) DO NOTHING;",+ )?;+ let mut prepared_insert_domain = tx.prepare(+ "INSERT INTO domain (domain, max_incoming_links, status)+ VALUES (?1, 0, ?2)+ ON CONFLICT (domain) DO NOTHING;",+ )?;++ for url in &response.discovered_urls {+ if domain.domain() != url.domain() {+ prepared_diff_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;+ } else {+ prepared_same_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;+ }++ prepared_insert_domain.execute((url.domain(), DomainStatus::Pending))?;+ }++ let mut prepared_update_status_url =+ tx.prepare("UPDATE url SET status = ?1, error_code = ?2 WHERE url = ?3;")?;+ let mut prepared_insert_redirect = tx.prepare(+ "INSERT INTO redirect (from_url, to_url)+ VALUES (?1, ?2)+ ON CONFLICT (from_url, to_url) DO NOTHING;",+ )?;++ // update status of URLs+ for url_res in &response.url_responses {+ match url_res {+ UrlResponse::Success { url } => {+ prepared_update_status_url.execute((+ UrlStatus::Done,+ None::<u16>,+ url.full(),+ ))?;+ }+ UrlResponse::Failed { url, status_code } => {+ prepared_update_status_url.execute((+ UrlStatus::Failed,+ status_code,+ url.full(),+ ))?;+ }+ UrlResponse::Redirected { url, new_url } => {+ prepared_update_status_url.execute((+ UrlStatus::Done,+ None::<u16>,+ url.full(),+ ))?;+ prepared_insert_redirect.execute((url.full(), new_url.full()))?;+ prepared_same_domain.execute((+ new_url.full(),+ new_url.domain(),+ UrlStatus::Done,+ 0,+ ))?;+ }+ }+ }++ tx.execute(+ "UPDATE domain SET status = ?1 WHERE domain = ?2;",+ (DomainStatus::Pending, &response.domain.0),+ )?;+ }++ tx.commit()?;++ let mut domains = HashSet::new();+ domains.insert(response.domain.clone());+ domains.extend(+ response+ .discovered_urls+ .iter()+ .map(|url| Domain(url.domain().to_string())),+ );++ self.update_max_inlinks_domains(domains.iter())?;++ Ok(())+ }++ fn update_max_inlinks_domains<'a, I: Iterator<Item = &'a Domain>>(+ &self,+ domains: I,+ ) -> Result<()> {+ let tx = self.conn.unchecked_transaction()?;++ {+ // update max_incoming_links for domain based on+ // max_incoming_links for URLs in domain.+ // if no pending URLs in domain, set max_incoming_links to 0+ let mut prepared = tx.prepare(+ "UPDATE domain SET max_incoming_links = IFNULL((+ SELECT MAX(incoming_links) FROM url+ WHERE domain = ?1 AND status = ?2+ ), 0) WHERE domain = ?1;",+ )?;++ for domain in domains {+ prepared.execute((&domain.0, UrlStatus::Pending))?;+ }+ }++ tx.commit()?;++ Ok(())+ }++ pub fn is_done(&self) -> bool {+ self.num_crawled_urls.load(Ordering::SeqCst) >= self.num_urls_to_crawl+ }++ pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {+ let start = Instant::now();+ let tx = self.conn.unchecked_transaction()?;+ let mut jobs = Vec::new();++ {+ // weighted sample from domains that are not currently being crawled+ // see https://www.kaggle.com/code/kotamori/random-sample-with-weights-on-sql/notebook for details on math+ let mut stmt = tx.prepare(+ "SELECT domain, -log((abs(random()) % 1000000 + 0.5) / 1000000.0) / (max_incoming_links + 1) as priority FROM domain+ WHERE status = ?1+ ORDER BY priority DESC+ LIMIT ?2;",+ )?;++ // dfs sample+ // let mut stmt = tx.prepare(+ // "SELECT domain FROM domain+ // WHERE status = ?1+ // LIMIT ?2;",+ // )?;++ let rows = stmt.query_map((DomainStatus::Pending, num_jobs), |row| {+ row.get::<_, String>(0)+ })?;++ let mut domains = Vec::new();+ for domain in rows {+ domains.push(Domain(domain?));+ }++ tracing::debug!("sampled domains: {:?}", domains);++ // get URLs from sampled domains+ let mut stmt = tx.prepare(+ "SELECT url FROM url+ WHERE domain = ?1 AND status = ?2+ ORDER BY incoming_links DESC+ LIMIT ?3;",+ )?;++ let mut prepared_update_status_url =+ tx.prepare("UPDATE url SET status = ?1 WHERE url = ?2;")?;+ let mut prepared_update_status_domain =+ tx.prepare("UPDATE domain SET status = ?1 WHERE domain = ?2;")?;++ for domain in domains {+ let mut urls = Vec::new();++ let rows = stmt+ .query_map((&domain.0, UrlStatus::Pending, DEFAULT_JOB_URLS), |row| {+ row.get::<_, String>(0)+ })?;++ prepared_update_status_domain+ .execute((DomainStatus::CrawlInProgress, &domain.0))?;++ for url in rows {+ let url = url?;++ prepared_update_status_url.execute((UrlStatus::Crawling, url.clone()))?;++ urls.push(url.into());+ }++ jobs.push(Job {+ domain,+ fetch_sitemap: true, // TODO: only fetch sitemap if we haven't already+ urls: urls.into(),+ });+ }+ }+ tx.commit()?;++ tracing::debug!("sampled jobs: {:?}", jobs);+ tracing::info!("sampled {} jobs in {:?}", jobs.len(), start.elapsed());++ Ok(jobs)+ }+}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsnew file mode 100644index 00000000..b7fb44b1--- /dev/null+++ b/core/src/crawler/mod.rs@@ -0,0 +1,185 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ collections::{HashMap, VecDeque},+ sync::Arc,+ time::Duration,+};++use tokio::sync::Mutex;++use crate::{webpage::Url, CrawlerConfig};++use self::{+ robots_txt::RobotsTxtManager,+ warc_writer::{WarcWriter, WarcWriterMessage},+ worker::Worker,+};++mod coordinator;+mod robots_txt;+mod warc_writer;+mod worker;++pub use coordinator::CrawlCoordinator;++const DEFAULT_POLITENESS_FACTOR: f32 = 1.0;++#[derive(thiserror::Error, Debug)]+pub enum Error {+ #[error("reqwest error: {0}")]+ Reqwest(#[from] reqwest::Error),++ #[error("chrono out-of-range error: {0}")]+ Chrono(#[from] chrono::OutOfRangeError),++ #[error("invalid content type: {0}")]+ InvalidContentType(String),++ #[error("fetch failed")]+ FetchFailed(reqwest::StatusCode),++ #[error("content too large")]+ ContentTooLarge,++ #[error("invalid politeness factor")]+ InvalidPolitenessFactor,++ #[error("send error")]+ SendError(#[from] tokio::sync::mpsc::error::SendError<WarcWriterMessage>),++ #[error("sqlite error")]+ Sqlite(#[from] rusqlite::Error),++ #[error("addr parse error: {0}")]+ AddrParse(#[from] std::net::AddrParseError),+}++type Result<T> = std::result::Result<T, Error>;++#[derive(Clone, Debug, PartialEq, Eq, Hash)]+struct Site(String);++#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Domain(String);++/// All urls in a job must be from the same domain and only one job per domain.+/// at a time. This ensures that we stay polite when crawling.+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]+pub struct Job {+ pub domain: Domain,+ pub fetch_sitemap: bool,+ pub urls: VecDeque<Url>,+}++#[derive(Debug, serde::Serialize, serde::Deserialize)]+enum UrlResponse {+ Success { url: Url },+ Failed { url: Url, status_code: Option<u16> },+ Redirected { url: Url, new_url: Url },+}++#[derive(Debug, serde::Serialize, serde::Deserialize)]+pub struct JobResponse {+ domain: Domain,+ url_responses: Vec<UrlResponse>,+ discovered_urls: Vec<Url>,+}++struct WorkerJob {+ robotstxt: RobotsTxtManager,+ job: Job,+}++impl From<Job> for WorkerJob {+ fn from(value: Job) -> Self {+ Self {+ robotstxt: RobotsTxtManager::new(),+ job: value,+ }+ }+}++pub enum Command {+ Job(Job),+ Shutdown,+}++#[derive(Debug, Clone)]+pub struct CrawlDatum {+ url: Url,+ status_code: u16,+ headers: HashMap<String, String>,+ body: String,+ fetch_time_ms: u64,+}++pub struct Crawler {+ writer: Arc<WarcWriter>,+ handles: Vec<tokio::task::JoinHandle<()>>,+}++impl Crawler {+ pub async fn new(config: CrawlerConfig) -> Result<Self> {+ let pending_commands = Arc::new(Mutex::new(VecDeque::new()));+ let writer = Arc::new(WarcWriter::new(config.s3));+ let timeout = Duration::from_secs(config.timeout_seconds);+ let mut handles = Vec::new();+ let coordinator_host = config.coordinator_host.parse()?;++ for _ in 0..config.num_workers {+ let worker = Worker::new(+ Arc::clone(&pending_commands),+ Arc::clone(&writer),+ config.user_agent.clone(),+ config+ .politeness_factor+ .unwrap_or(DEFAULT_POLITENESS_FACTOR),+ timeout,+ coordinator_host,+ config.num_workers,+ )?;++ handles.push(tokio::spawn(async move {+ worker.run().await;+ }));+ }++ Ok(Self { writer, handles })+ }++ pub async fn wait(self) {+ for handle in self.handles {+ handle.await.unwrap();+ }++ self.writer.finish().await.unwrap();+ }+}++#[derive(serde::Serialize, serde::Deserialize)]+pub enum Request {+ NewJobs { num_jobs: usize },+ CrawlResult { job_response: JobResponse },+}++#[derive(serde::Serialize, serde::Deserialize)]+pub enum Response {+ NewJobs { jobs: Vec<Job> },+ Done,+ Ok,+}diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsnew file mode 100644index 00000000..77246918--- /dev/null+++ b/core/src/crawler/robots_txt.rs@@ -0,0 +1,121 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{collections::HashMap, time::Instant};++use robotstxt_with_cache::matcher::{+ CachingRobotsMatcher, LongestMatchRobotsMatchStrategy, RobotsMatcher,+};++use crate::webpage::Url;++use super::{Error, Result, Site};++pub struct RobotsTxtManager {+ cache: HashMap<Site, Option<RobotsTxt>>, // None if robots.txt does not exist+}++impl RobotsTxtManager {+ pub fn new() -> Self {+ Self {+ cache: HashMap::new(),+ }+ }++ pub async fn is_allowed(&mut self, url: &Url, user_agent: &str) -> bool {+ match self.get_mut(url).await {+ Ok(Some(robots_txt)) => robots_txt+ .matcher+ .one_agent_allowed_by_robots(user_agent, &url.full()),+ _ => true,+ }+ }++ async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {+ let res = reqwest::get(&format!("https://{}/robots.txt", site.0)).await?;++ if res.status() != reqwest::StatusCode::OK {+ return Err(Error::FetchFailed(res.status()));+ }++ let body = res.text().await?;++ RobotsTxt::new(body)+ }++ async fn get_mut(&mut self, url: &Url) -> Result<Option<&mut RobotsTxt>> {+ let site = Site(url.site().to_string());++ let should_fetch = match self.cache.get(&site) {+ Some(Some(robots_txt)) => Instant::now() > robots_txt.valid_until,+ Some(None) => false, // robots.txt does not exist+ None => true,+ };++ if should_fetch {+ match self.fetch_robots_txt(&site).await {+ Ok(robots_txt) => {+ self.cache.insert(site.clone(), Some(robots_txt));+ }+ Err(Error::FetchFailed(status)) if status == reqwest::StatusCode::NOT_FOUND => {+ self.cache.insert(site.clone(), None);+ }+ Err(err) => {+ tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);+ }+ }+ }++ match self.cache.get_mut(&site) {+ Some(Some(robot)) => Ok(Some(robot)),+ _ => Ok(None),+ }+ }++ pub async fn sitemap(&mut self, url: &Url) -> Result<Option<Url>> {+ Ok(self+ .get_mut(url)+ .await?+ .and_then(|robots_txt| robots_txt.sitemap.clone()))+ }+}++struct RobotsTxt {+ valid_until: Instant,+ matcher: CachingRobotsMatcher<LongestMatchRobotsMatchStrategy>,+ sitemap: Option<Url>,+}++impl RobotsTxt {+ fn new(body: String) -> Result<Self> {+ let mut matcher = CachingRobotsMatcher::new(RobotsMatcher::default());++ matcher.parse(&body);++ let sitemap = body+ .to_ascii_lowercase()+ .lines()+ .find(|line| line.starts_with("sitemap:"))+ .map(|line| line.split(':').nth(1).unwrap().trim().to_string())+ .map(Url::from);++ Ok(Self {+ valid_until: Instant::now() + chrono::Duration::hours(24).to_std()?,+ matcher,+ sitemap,+ })+ }+}diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsnew file mode 100644index 00000000..d4f544f8--- /dev/null+++ b/core/src/crawler/warc_writer.rs@@ -0,0 +1,142 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{warc, S3Config};++use super::{CrawlDatum, Result};++/// The WarcWriter is responsible for storing the crawl datums+/// as WARC files on S3.+pub struct WarcWriter {+ tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,+}++#[derive(Debug, Clone)]+pub enum WarcWriterMessage {+ Crawl(CrawlDatum),+ Finish,+}++async fn commit(writer: warc::WarcWriter, s3: S3Config) {+ let filename = format!(+ "{}_{}.warc.gz",+ chrono::Utc::now().to_rfc3339(),+ uuid::Uuid::new_v4()+ );+ let data = writer.finish().unwrap();++ match s3::Bucket::new(+ &s3.bucket,+ s3::Region::Custom {+ region: "".to_string(),+ endpoint: s3.endpoint.clone(),+ },+ s3::creds::Credentials {+ access_key: Some(s3.access_key.clone()),+ secret_key: Some(s3.secret_key.clone()),+ security_token: None,+ session_token: None,+ expiration: None,+ },+ ) {+ Ok(bucket) => {+ let bucket = bucket.with_path_style();++ if let Err(err) = bucket+ .put_object_with_content_type(+ &format!("{}/{}", &s3.folder, filename),+ &data,+ "application/warc",+ )+ .await+ {+ tracing::error!("failed to upload to bucket: {:?}", err);+ }+ }+ Err(err) => tracing::error!("failed to connect to bucket: {:?}", err),+ }+}++async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3: S3Config) {+ let mut writer = warc::WarcWriter::new();++ while let Some(message) = rx.recv().await {+ match message {+ WarcWriterMessage::Crawl(datum) => {+ let is_pdf = datum+ .headers+ .get("content-type")+ .map(|ct| ct.contains("application/pdf"))+ .unwrap_or(false);++ let payload = if is_pdf {+ Some("application/pdf".to_string())+ } else {+ None+ };++ let warc_record = warc::WarcRecord {+ request: warc::Request {+ url: datum.url.full(),+ },+ response: warc::Response {+ body: datum.body,+ payload_type: payload,+ },+ metadata: warc::Metadata {+ fetch_time_ms: datum.fetch_time_ms as usize,+ },+ };++ writer.write(&warc_record).unwrap();++ if writer.num_bytes() > 1_000_000_000 {+ commit(writer, s3.clone()).await;+ writer = warc::WarcWriter::new();+ }+ }+ WarcWriterMessage::Finish => {+ if writer.num_writes() > 0 {+ commit(writer, s3.clone()).await;+ }+ break;+ }+ }+ }+}++impl WarcWriter {+ pub fn new(s3: S3Config) -> Self {+ let (tx, rx) = tokio::sync::mpsc::channel(1000);++ tokio::spawn(writer_task(rx, s3));++ Self { tx }+ }++ pub async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {+ self.tx.send(WarcWriterMessage::Crawl(crawl_datum)).await?;++ Ok(())+ }++ pub async fn finish(&self) -> Result<()> {+ self.tx.send(WarcWriterMessage::Finish).await?;+ self.tx.closed().await;++ Ok(())+ }+}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsnew file mode 100644index 00000000..83a3aadc--- /dev/null+++ b/core/src/crawler/worker.rs@@ -0,0 +1,477 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use futures::{future::BoxFuture, FutureExt};+use quick_xml::events::Event;++use std::{+ collections::{HashMap, HashSet, VecDeque},+ net::SocketAddr,+ sync::Arc,+ time::{Duration, Instant},+};++use tokio::sync::Mutex;++use crate::{+ crawler::{JobResponse, Request, Response},+ distributed::{retry_strategy::ExponentialBackoff, sonic},+ webpage::{Html, Url},+};++use super::{Command, CrawlDatum, Error, Result, Site, UrlResponse, WarcWriter, WorkerJob};++const MIN_CRAWL_DELAY_MS: u64 = 200;+const MAX_CRAWL_DELAY_MS: u64 = 30_000;+const MAX_POLITENESS_FACTOR: f32 = 128.0;+const MIN_POLITENESS_FACTOR: f32 = 1.0;++pub struct Worker {+ current_job: Option<WorkerJob>,+ pending_commands: Arc<Mutex<VecDeque<Command>>>,+ writer: Arc<WarcWriter>,+ client: reqwest::Client,+ user_agent: String,+ default_politeness_factor: f32,+ politeness_factor: f32,+ coordinator_host: SocketAddr,+ num_jobs_per_fetch: usize,+}++impl Worker {+ pub fn new(+ pending_commands: Arc<Mutex<VecDeque<Command>>>,+ writer: Arc<WarcWriter>,+ user_agent: String,+ politeness_factor: f32,+ timeout: Duration,+ coordinator_host: SocketAddr,+ num_jobs_per_fetch: usize,+ ) -> Result<Self> {+ if politeness_factor < MIN_POLITENESS_FACTOR {+ return Err(Error::InvalidPolitenessFactor);+ }++ Ok(Self {+ writer,+ current_job: None,+ pending_commands,+ client: reqwest::Client::builder()+ .timeout(timeout)+ .connect_timeout(timeout)+ .http2_keep_alive_interval(None)+ .user_agent(&user_agent)+ .build()?,+ user_agent,+ politeness_factor,+ default_politeness_factor: politeness_factor,+ coordinator_host,+ num_jobs_per_fetch,+ })+ }++ fn coordinator_conn(&self) -> sonic::ResilientConnection<std::iter::Take<ExponentialBackoff>> {+ let retry = ExponentialBackoff::from_millis(1_000)+ .with_limit(Duration::from_millis(20_000))+ .take(5);++ sonic::ResilientConnection::create(self.coordinator_host, retry)+ }++ pub async fn run(mut self) {+ loop {+ let mut guard = self.pending_commands.lock().await;+ let command = guard.pop_front();++ if let Some(command) = command {+ drop(guard); // let other workers get jobs++ match command {+ Command::Job(job) => {+ self.current_job = Some(job.into());+ self.process_job().await;+ }+ Command::Shutdown => {+ self.pending_commands+ .lock()+ .await+ .push_back(Command::Shutdown);++ break;+ }+ }+ } else {+ let mut conn = self.coordinator_conn();+ let res = conn+ .send_with_timeout::<_, Response>(+ &Request::NewJobs {+ num_jobs: self.num_jobs_per_fetch,+ },+ Duration::from_secs(60),+ )+ .await;++ match res {+ Ok(sonic::Response::Content(Response::NewJobs { jobs })) => {+ if jobs.is_empty() {+ drop(guard);+ tokio::time::sleep(Duration::from_secs(30)).await;+ continue;+ }++ guard.extend(jobs.into_iter().map(Command::Job));+ }+ Ok(sonic::Response::Content(Response::Done)) => {+ guard.push_back(Command::Shutdown);++ break;+ }+ _ => {+ drop(guard);+ tokio::time::sleep(Duration::from_secs(30)).await;+ continue;+ }+ }+ drop(guard)+ }+ }+ }++ async fn process_job(&mut self) {+ self.politeness_factor = self.default_politeness_factor;++ let mut job = self.current_job.take().unwrap();+ tracing::info!("Processing job: {:?}", job.job.domain);++ let mut url_responses: Vec<UrlResponse> = Vec::new();+ let mut discovered_urls: HashSet<Url> = HashSet::new();++ let mut crawled_sitemaps: HashSet<Site> = HashSet::new();++ let num_urls = job.job.urls.len();+ for (i, url) in job.job.urls.into_iter().enumerate() {+ if !job.robotstxt.is_allowed(&url, &self.user_agent).await {+ continue;+ }++ let site = Site(url.site().to_string());+ if job.job.fetch_sitemap && !crawled_sitemaps.contains(&site) {+ crawled_sitemaps.insert(site.clone());++ if let Ok(Some(sitemap)) = job.robotstxt.sitemap(&url).await {+ let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;+ discovered_urls.extend(sitemap_urls.into_iter());+ }+ }++ let res = self.process_url(url).await;++ discovered_urls.extend(res.new_urls.into_iter());+ url_responses.push(res.response);++ let mut delay = res.fetch_time.mul_f32(self.politeness_factor);++ if delay < Duration::from_millis(MIN_CRAWL_DELAY_MS) {+ delay = Duration::from_millis(MIN_CRAWL_DELAY_MS);+ }++ if delay > Duration::from_millis(MAX_CRAWL_DELAY_MS) {+ delay = Duration::from_millis(MAX_CRAWL_DELAY_MS);+ }++ if i < num_urls - 1 {+ tokio::time::sleep(delay).await;+ }+ }++ // send response to coordinator+ let job_response = JobResponse {+ domain: job.job.domain,+ url_responses,+ discovered_urls: discovered_urls.into_iter().collect(),+ };++ let mut conn = self.coordinator_conn();+ let _ = conn+ .send_with_timeout::<_, Response>(+ &Request::CrawlResult { job_response },+ Duration::from_secs(60),+ )+ .await+ .expect("Failed to send crawl result to coordinator");+ }++ async fn process_url(&mut self, url: Url) -> ProcessedUrl {+ let fetch = self.crawl_url(url.clone()).await;++ match fetch {+ Ok(datum) => {+ if matches!(datum.status_code, 200 | 301 | 302) {+ self.save_datum(datum.clone()).await;++ let html = Html::parse(&datum.body, &datum.url.full());+ let new_urls = html+ .all_links()+ .into_iter()+ .map(|link| Url::from(link.destination.full()).into_absolute(&datum.url))+ .filter(|url| {+ !url.path_ends_with(".pdf")+ && !url.path_ends_with(".jpg")+ && !url.path_ends_with(".zip")+ && !url.path_ends_with(".png")+ && !url.path_ends_with(".css")+ && !url.path_ends_with(".js")+ && !url.path_ends_with(".json")+ && !url.path_ends_with(".jsonp")+ && !url.path_ends_with(".woff2")+ && !url.path_ends_with(".woff")+ && !url.path_ends_with(".ttf")+ && !url.path_ends_with(".svg")+ && !url.path_ends_with(".gif")+ && !url.path_ends_with(".jpeg")+ && !url.path_ends_with(".ico")+ && !url.path_ends_with(".mp4")+ && !url.path_ends_with(".mp3")+ && !url.path_ends_with(".avi")+ && !url.path_ends_with(".mov")+ && !url.path_ends_with(".mpeg")+ && !url.path_ends_with(".webm")+ && !url.path_ends_with(".wav")+ && !url.path_ends_with(".flac")+ && !url.path_ends_with(".aac")+ && !url.path_ends_with(".ogg")+ && !url.path_ends_with(".m4a")+ && !url.path_ends_with(".m4v")+ })+ .collect();++ let url_res =+ if datum.status_code == 301 || datum.status_code == 302 || url != datum.url+ {+ UrlResponse::Redirected {+ url,+ new_url: datum.url,+ }+ } else {+ UrlResponse::Success { url: datum.url }+ };++ ProcessedUrl {+ new_urls,+ response: url_res,+ fetch_time: Duration::from_millis(datum.fetch_time_ms),+ }+ } else {+ if datum.status_code == 429 {+ self.politeness_factor *= 2.0;++ if self.politeness_factor > MAX_POLITENESS_FACTOR {+ self.politeness_factor = MAX_POLITENESS_FACTOR;+ }+ }+ ProcessedUrl {+ new_urls: Vec::new(),+ response: UrlResponse::Failed {+ url,+ status_code: Some(datum.status_code),+ },+ fetch_time: Duration::from_millis(datum.fetch_time_ms),+ }+ }+ }+ Err(err) => {+ tracing::warn!("failed to fetch url: {}", err);++ ProcessedUrl {+ new_urls: Vec::new(),+ response: UrlResponse::Failed {+ url,+ status_code: None,+ },+ fetch_time: Duration::from_millis(0),+ }+ }+ }+ }++ async fn save_datum(&self, datum: CrawlDatum) {+ self.writer.write(datum).await.ok();+ }++ async fn fetch(&self, url: Url) -> Result<reqwest::Response> {+ let backoff = ExponentialBackoff::from_millis(1000)+ .with_limit(Duration::from_secs(20))+ .take(3);++ let mut res = Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT));+ for time in backoff {+ if let Ok(cur_res) = self.client.get(url.full()).send().await {+ res = Ok(cur_res);+ break;+ } else {+ tokio::time::sleep(time).await;+ }+ }++ res+ }++ async fn crawl_url(&self, url: Url) -> Result<CrawlDatum> {+ let start = Instant::now();+ let res = self.fetch(url.clone()).await?;+ let fetch_time = start.elapsed();++ let headers: HashMap<_, _> = res+ .headers()+ .iter()+ .filter_map(|(k, v)| {+ let k = k.to_string();+ if let Ok(v) = v.to_str() {+ return Some((k, v.to_string()));+ }++ None+ })+ .collect();++ // check if content length is too large+ if let Some(content_length) = headers.get("content-length") {+ if content_length.parse::<usize>().unwrap_or(0) > 10_000_000 {+ return Err(Error::ContentTooLarge);+ }+ }++ // check if content type is html+ if let Some(content_type) = headers.get("content-type") {+ if !content_type.contains("text/html") {+ return Err(Error::InvalidContentType(content_type.to_string()));+ }+ }++ let status_code = res.status().as_u16();++ let res_url = res.url().as_str().to_string();+ let body = res.text().await?;++ Ok(CrawlDatum {+ url: Url::from(res_url).full().into(),+ status_code,+ headers,+ body,+ fetch_time_ms: fetch_time.as_millis() as u64,+ })+ }++ fn urls_from_sitemap(+ &self,+ sitemap: Url,+ depth: usize,+ max_depth: usize,+ ) -> BoxFuture<'_, Vec<Url>> {+ async move {+ if depth == max_depth {+ return vec![];+ }++ // fetch url+ let res = self.fetch(sitemap).await;++ if res.is_err() {+ return vec![];+ }+ let res = res.unwrap();++ if res.status() != reqwest::StatusCode::OK {+ return vec![];+ }++ let body = res.text().await;++ if body.is_err() {+ return vec![];+ }++ let body = body.unwrap();++ // parse xml+ let mut reader = quick_xml::Reader::from_str(&body);+ let mut buf = Vec::new();++ let mut urls = vec![];++ let mut in_sitemap = false;+ let mut in_url = false;+ let mut in_loc = false;++ loop {+ match reader.read_event(&mut buf) {+ Ok(Event::Start(ref e)) => {+ if e.name() == b"sitemap" {+ in_sitemap = true;+ } else if e.name() == b"url" {+ in_url = true;+ } else if e.name() == b"loc" {+ in_loc = true;+ }+ }+ Ok(Event::End(ref e)) => {+ if e.name() == b"sitemap" {+ in_sitemap = false;+ } else if e.name() == b"url" {+ in_url = false;+ } else if e.name() == b"loc" {+ in_loc = false;+ }+ }+ Ok(Event::Text(e)) => {+ if in_sitemap && in_loc {+ let url = Url::from(e.unescape_and_decode(&reader).unwrap())+ .full()+ .into();++ tokio::time::sleep(Duration::from_millis(MIN_CRAWL_DELAY_MS)).await;++ urls.append(+ &mut self.urls_from_sitemap(url, depth + 1, max_depth).await,+ );+ } else if in_url && in_loc {+ let url = Url::from(e.unescape_and_decode(&reader).unwrap())+ .full()+ .into();+ urls.push(url);+ }+ }+ Ok(Event::Eof) => break,+ Err(e) => {+ tracing::warn!("failed to parse sitemap: {}", e);+ break;+ }+ _ => (),+ }++ buf.clear();+ }++ urls+ }+ .boxed()+ }+}++struct ProcessedUrl {+ new_urls: Vec<Url>,+ response: UrlResponse,+ fetch_time: Duration,+}diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsnew file mode 100644index 00000000..f3be97ea--- /dev/null+++ b/core/src/entrypoint/crawler.rs@@ -0,0 +1,73 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use crate::{+ crawler::{CrawlCoordinator, Crawler},+ distributed::sonic,+ CrawlCoordinatorConfig, CrawlerConfig, Result,+};++pub async fn worker(config: CrawlerConfig) -> Result<()> {+ let crawler = Crawler::new(config).await?;++ crawler.wait().await;++ Ok(())+}++pub async fn coordinator(config: CrawlCoordinatorConfig) -> Result<()> {+ let coordinator = CrawlCoordinator::new(+ config.crawldb_folder,+ config.num_urls_to_crawl,+ config.seed_urls,+ )?;++ let addr: SocketAddr = config.host;+ let server = sonic::Server::bind(addr).await.unwrap();++ tracing::info!("Crawl coordinator listening on {}", addr);++ loop {+ if let Ok(req) = server.accept::<crate::crawler::Request>().await {+ match &req.body {+ crate::crawler::Request::NewJobs { num_jobs } => {+ if coordinator.is_done() {+ req.respond(sonic::Response::Content(crate::crawler::Response::Done))+ .await+ .ok();+ } else {+ let jobs = coordinator.sample_jobs(*num_jobs)?;++ req.respond(sonic::Response::Content(+ crate::crawler::Response::NewJobs { jobs },+ ))+ .await+ .ok();+ }+ }+ crate::crawler::Request::CrawlResult { job_response } => {+ coordinator.add_response(job_response)?;++ req.respond(sonic::Response::Content(crate::crawler::Response::Ok))+ .await+ .ok();+ }+ }+ }+ }+}diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 3e38fece..12af3807 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -19,6 +19,7 @@ mod centrality;#[cfg(feature = "dev")]pub mod configure;pub mod crawl_stability;+pub mod crawler;pub mod dmoz_parser;mod entity;pub mod frontend;diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex ec1b4dc6..0dc440df 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -75,7 +75,7 @@ pub fn process_job(job: &Job) -> webgraph::Webgraph {for record in file.records().flatten() {let webpage = Html::parse_without_text(&record.response.body, &record.request.url);for link in webpage- .links()+ .anchor_links().into_iter().filter(|link| matches!(link.destination.protocol(), "http" | "https")).filter(|link| link.source.domain() != link.destination.domain())diff --git a/core/src/lib.rs b/core/src/lib.rsindex 3a21e077..df08eb1a 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -38,6 +38,7 @@ mod api;mod autosuggest;mod bangs;mod collector;+mod crawler;mod directory;mod distributed;mod entity_index;@@ -194,6 +195,32 @@ pub struct SearchServerConfig {pub host: SocketAddr,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct S3Config {+ pub bucket: String,+ pub folder: String,+ pub access_key: String,+ pub secret_key: String,+ pub endpoint: String,+}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct CrawlCoordinatorConfig {+ pub crawldb_folder: String,+ pub host: SocketAddr,+ pub num_urls_to_crawl: u64,+ pub seed_urls: Vec<String>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct CrawlerConfig {+ pub num_workers: usize,+ pub user_agent: String,+ pub politeness_factor: Option<f32>,+ pub timeout_seconds: u64,+ pub s3: S3Config,+ pub coordinator_host: String,+}+#[derive(Debug, Serialize, Deserialize, Clone)]#[serde(tag = "type", rename_all = "snake_case")]pub enum AcceleratorDevice {@@ -337,6 +364,9 @@ pub enum Error {#[error("Fact check model")]FactCheckModel(#[from] crate::fact_check_model::Error),++ #[error("Crawler")]+ Crawler(#[from] crate::crawler::Error),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/core/src/main.rs b/core/src/main.rsindex 6292fe63..1ef81fa3 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -24,7 +24,10 @@ use stract::entrypoint::configure;use stract::entrypoint::indexer::IndexPointer;use stract::entrypoint::{self, frontend, search_server, webgraph_server};use stract::webgraph::WebgraphBuilder;-use stract::{AliceLocalConfig, FrontendConfig, SearchServerConfig, WebgraphServerConfig};+use stract::{+ AliceLocalConfig, CrawlCoordinatorConfig, CrawlerConfig, FrontendConfig, SearchServerConfig,+ WebgraphServerConfig,+};use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -83,6 +86,12 @@ enum Commands {online_harmonic_path: String,output_path: String,},+ CrawlCoordinator {+ config_path: String,+ },+ Crawler {+ config_path: String,+ },#[cfg(feature = "dev")]Configure {#[clap(long, takes_value = false)]@@ -326,6 +335,22 @@ fn main() -> Result<()> {}AliceOptions::GenerateKey => entrypoint::alice::generate_key(),},+ Commands::CrawlCoordinator { config_path } => {+ let config: CrawlCoordinatorConfig = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::crawler::coordinator(config))?+ }+ Commands::Crawler { config_path } => {+ let config: CrawlerConfig = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::crawler::worker(config))?+ }}Ok(())diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 38b59335..b6c9b866 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -882,7 +882,7 @@ mod tests {.webpages;assert_eq!(res.len(), 1);- assert_eq!(res[0].url, "https://www.b.com/");+ assert_eq!(res[0].url, "https://www.b.com");let res = searcher.search(&SearchQuery {@@ -906,7 +906,7 @@ mod tests {.webpages;assert_eq!(res.len(), 1);- assert_eq!(res[0].url, "https://www.b.com/");+ assert_eq!(res[0].url, "https://www.b.com");let res = searcher.search(&SearchQuery {@@ -930,7 +930,7 @@ mod tests {.webpages;assert_eq!(res.len(), 1);- assert_eq!(res[0].url, "https://www.a.com/");+ assert_eq!(res[0].url, "https://www.a.com");let res = searcher.search(&SearchQuery {@@ -954,7 +954,7 @@ mod tests {.webpages;assert_eq!(res.len(), 1);- assert_eq!(res[0].url, "https://www.b.com/");+ assert_eq!(res[0].url, "https://www.b.com");}#[test]diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 31746161..48420441 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -130,7 +130,7 @@ impl<M: CrossEncoder> ReRanker<M> {let website = website.as_mut_ranking();let title = website.title.clone().unwrap_or_default();let body = website.clean_body.clone().unwrap_or_default();- let text = title + ". " + &body;+ let text = title + ". " + body.as_str();bodies.push(text);}diff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rsindex 163bd706..516d153c 100644--- a/core/src/search_prettifier/stack_overflow.rs+++ b/core/src/search_prettifier/stack_overflow.rs@@ -92,7 +92,7 @@ fn parse_so_answer(.and_then(|prop| prop.try_into_string()).map(Url::from).map(|mut url| {- url.prefix_with(&Url::from(webpage_url.site()));+ url = url.into_absolute(&Url::from(webpage_url.site()));url})?;diff --git a/core/src/warc.rs b/core/src/warc.rsindex 7cdf9724..19956f50 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -24,6 +24,9 @@ use std::thread::sleep;use std::time::Duration;use flate2::read::MultiGzDecoder;+use flate2::write::GzEncoder;+use flate2::Compression;+use regex::Regex;use tracing::{debug, trace};pub(crate) struct WarcFile {@@ -171,16 +174,16 @@ struct RawWarcRecord {}#[derive(Debug)]-pub(crate) struct WarcRecord {- pub(crate) request: Request,- pub(crate) response: Response,- pub(crate) metadata: Metadata,+pub struct WarcRecord {+ pub request: Request,+ pub response: Response,+ pub metadata: Metadata,}#[derive(Debug)]-pub(crate) struct Request {+pub struct Request {// WARC-Target-URI- pub(crate) url: String,+ pub url: String,}impl Request {@@ -196,9 +199,9 @@ impl Request {}#[derive(Debug)]-pub(crate) struct Response {- pub(crate) body: String,- pub(crate) payload_type: Option<String>,+pub struct Response {+ pub body: String,+ pub payload_type: Option<String>,}impl Response {@@ -217,9 +220,9 @@ impl Response {}#[derive(Debug)]-pub(crate) struct Metadata {+pub struct Metadata {// fetchTimeMs- pub(crate) fetch_time_ms: usize,+ pub fetch_time_ms: usize,}impl Metadata {@@ -243,7 +246,7 @@ impl Metadata {}}-pub(crate) struct RecordIterator<R: Read> {+pub struct RecordIterator<R: Read> {reader: BufReader<MultiGzDecoder<R>>,num_reads: usize,}@@ -389,6 +392,103 @@ impl<R: Read> Iterator for RecordIterator<R> {}}+pub struct WarcWriter {+ num_writes: usize,+ writer: GzEncoder<Vec<u8>>,+}++impl WarcWriter {+ pub fn new() -> Self {+ let mut writer = GzEncoder::new(Vec::new(), Compression::best());++ writer.write_all("WARC/1.0\r\n".as_bytes()).unwrap();+ writer+ .write_all("WARC-Type: warcinfo\r\n".as_bytes())+ .unwrap();++ let date = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();+ let content = format!("ISPARTOF: crawl[{}]", date);+ let content_len = content.len();++ writer+ .write_all(format!("Content-Length: {content_len}\r\n").as_bytes())+ .unwrap();+ writer.write_all("\r\n".as_bytes()).unwrap();+ writer.write_all(content.as_bytes()).unwrap();+ writer.write_all("\r\n\r\n".as_bytes()).unwrap();++ Self {+ writer,+ num_writes: 0,+ }+ }++ pub fn write(&mut self, record: &WarcRecord) -> Result<()> {+ self.writer.write_all("WARC/1.0\r\n".as_bytes())?;++ self.writer.write_all("WARC-Type: request\r\n".as_bytes())?;+ self.writer+ .write_all(format!("WARC-Target-URI: {}\r\n", record.request.url).as_bytes())?;+ self.writer.write_all("Content-Length: 0\r\n".as_bytes())?;+ self.writer.write_all("\r\n".as_bytes())?;+ self.writer.write_all("\r\n\r\n".as_bytes())?;++ self.writer.write_all("WARC/1.0\r\n".as_bytes())?;+ self.writer+ .write_all("WARC-Type: response\r\n".as_bytes())?;++ if let Some(payload_type) = &record.response.payload_type {+ self.writer.write_all(+ format!("WARC-Identified-Payload-Type: {}\r\n", payload_type).as_bytes(),+ )?;+ }++ let content_len = record.response.body.len() + 4; // +4 is for the \r\n\r\n between http header and body+ self.writer+ .write_all(format!("Content-Length: {content_len}\r\n").as_bytes())?;++ self.writer.write_all("\r\n".as_bytes())?;+ // write the http-header here if we want to in the future+ self.writer.write_all("\r\n\r\n".as_bytes())?;++ let re = Regex::new(r"(\r\n\r\n)+").unwrap();+ re.replace_all(&record.response.body, "");+ let body = re.replace(&record.response.body, "");++ self.writer.write_all(body.as_bytes())?;+ self.writer.write_all("\r\n\r\n".as_bytes())?;++ self.writer.write_all("WARC/1.0\r\n".as_bytes())?;+ self.writer+ .write_all("WARC-Type: metadata\r\n".as_bytes())?;++ let body = format!("fetchTimeMs: {}", record.metadata.fetch_time_ms);+ let content_len = body.len();++ self.writer+ .write_all(format!("Content-Length: {content_len}\r\n").as_bytes())?;+ self.writer.write_all("\r\n".as_bytes())?;+ self.writer.write_all(body.as_bytes())?;+ self.writer.write_all("\r\n\r\n".as_bytes())?;++ self.num_writes += 1;++ Ok(())+ }++ pub fn finish(self) -> Result<Vec<u8>> {+ self.writer.finish().map_err(Error::IOError)+ }++ pub fn num_bytes(&self) -> usize {+ self.writer.get_ref().len()+ }++ pub fn num_writes(&self) -> usize {+ self.num_writes+ }+}+#[cfg(test)]mod tests {use super::*;@@ -440,4 +540,44 @@ mod tests {assert_eq!(&records[0].response.body, "body of response");assert_eq!(records[0].metadata.fetch_time_ms, 937);}++ #[test]+ fn writer_reader_invariant() {+ let mut writer = WarcWriter::new();+ let record1 = WarcRecord {+ request: Request {+ url: "http://0575ls.cn/news-52300.htm".to_string(),+ },+ response: Response {+ body: "body of response".to_string(),+ payload_type: Some("text/html".to_string()),+ },+ metadata: Metadata { fetch_time_ms: 937 },+ };+ writer.write(&record1).unwrap();++ let record2 = WarcRecord {+ request: Request {+ url: "http://0575ls.cn/news-52300.htm".to_string(),+ },+ response: Response {+ body: "body of response".to_string(),+ payload_type: Some("text/html".to_string()),+ },+ metadata: Metadata { fetch_time_ms: 937 },+ };+ writer.write(&record2).unwrap();++ let compressed = writer.finish().unwrap();++ let records: Vec<WarcRecord> = WarcFile::new(compressed)+ .records()+ .map(|res| res.unwrap())+ .collect();++ assert_eq!(records.len(), 2);+ assert_eq!(&records[0].request.url, "http://0575ls.cn/news-52300.htm");+ assert_eq!(&records[0].response.body, "body of response");+ assert_eq!(records[0].metadata.fetch_time_ms, 937);+ }}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex a3fdd28a..744afffc 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -369,7 +369,7 @@ impl Html {self.clean_text = Html::calculate_clean_text(¶graphs, &self.lang.unwrap_or(Lang::Eng));}- fn hyperlinks(&self) -> Vec<Link> {+ pub fn anchor_links(&self) -> Vec<Link> {let mut links = Vec::new();let mut open_links = Vec::new();@@ -387,9 +387,14 @@ impl Html {if &element.name.local == "a" {if let Some((text, attributes)) = open_links.pop() {if let Some(dest) = attributes.borrow().get("href") {+ if dest.starts_with("mailto:") || dest.starts_with("tel:") {+ continue;+ }+links.push(Link {source: self.url.clone(),- destination: dest.to_string().into(),+ destination: Url::from(dest.to_string())+ .into_absolute(self.url()),text: text.trim().to_string(),});}@@ -422,7 +427,7 @@ impl Html {if let Some(dest) = attributes.borrow().get("href") {links.push(Link {source: self.url.clone(),- destination: dest.to_string().into(),+ destination: Url::from(dest.to_string()).into_absolute(self.url()),text: text.trim().to_string(),});}@@ -431,7 +436,7 @@ impl Html {links}- fn _links_tag(&self) -> Vec<Link> {+ fn links_tag(&self) -> Vec<Link> {let mut links = Vec::new();for node in self.root.select("link").unwrap() {@@ -439,7 +444,7 @@ impl Html {if let Some(href) = element.attributes.borrow().get("href") {links.push(Link {source: self.url.clone(),- destination: Url::from(href),+ destination: Url::from(href).into_absolute(self.url()),text: String::new(),})}@@ -449,7 +454,7 @@ impl Html {links}- fn _metadata_links(&self) -> Vec<Link> {+ fn metadata_links(&self) -> Vec<Link> {self.metadata().into_iter().filter_map(|metadata| {@@ -470,7 +475,7 @@ impl Html {if let Some(content) = metadata.get("content") {return Some(Link {source: self.url().clone(),- destination: Url::from(content.as_str()),+ destination: Url::from(content.as_str()).into_absolute(self.url()),text: String::new(),});}@@ -490,7 +495,7 @@ impl Html {if let Some(content) = metadata.get("content") {return Some(Link {source: self.url().clone(),- destination: Url::from(content.as_str()),+ destination: Url::from(content.as_str()).into_absolute(self.url()),text: String::new(),});}@@ -502,32 +507,31 @@ impl Html {.collect()}- pub fn links(&self) -> Vec<Link> {- // let mut links = self.hyperlinks();+ pub fn all_links(&self) -> Vec<Link> {+ let mut links = self.anchor_links();- // links.extend(self.scripts().into_iter().filter_map(|script| {- // match script.attributes.get("src") {- // Some(url) => {- // let script_url = Url::from(url.as_str());- // if self.url().domain() != script_url.domain() {- // Some(Link {- // source: self.url.clone(),- // destination: script_url,- // text: String::new(),- // })- // } else {- // None- // }- // }- // None => None,- // }- // }));+ links.extend(self.scripts().into_iter().filter_map(|script| {+ match script.attributes.get("src") {+ Some(url) => {+ let script_url = Url::from(url.as_str()).into_absolute(self.url());+ if self.url().domain() != script_url.domain() {+ Some(Link {+ source: self.url.clone(),+ destination: script_url,+ text: String::new(),+ })+ } else {+ None+ }+ }+ None => None,+ }+ }));- // links.extend(self.links_tag().into_iter());- // links.extend(self.metadata_links().into_iter());+ links.extend(self.links_tag().into_iter());+ links.extend(self.metadata_links().into_iter());- // links- self.hyperlinks()+ links}pub fn favicon(&self) -> Option<FaviconLink> {@@ -549,11 +553,7 @@ impl Html {};let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());- let mut link: Url = link.to_string().into();-- if !link.is_full_path() {- link.prefix_with(&self.url);- }+ let link = Url::from(link.to_string()).into_absolute(self.url());let favicon = FaviconLink {link,@@ -1033,17 +1033,17 @@ impl Html {for script in self.scripts() {if let Some(link) = script.attributes.get("src") {- links.push(link.to_string().into());+ links.push(Url::from(link.to_string()).into_absolute(self.url()));}for res in URL_REGEX.find_iter(&script.content) {- links.push(res.as_str().to_string().into());+ links.push(Url::from(res.as_str().to_string()).into_absolute(self.url()));}}for node in self.root.select("link").unwrap() {if let Some(link) = node.attributes.borrow().get("href") {- links.push(link.to_string().into());+ links.push(Url::from(link.to_string()).into_absolute(self.url()));}}@@ -1156,9 +1156,7 @@ impl Html {})}).map(|mut image| {- if !image.url.is_full_path() {- image.url.prefix_with(&self.url);- }+ image.url = image.url.into_absolute(&self.url);image})@@ -1277,8 +1275,9 @@ mod tests {<meta name="meta1" content="value"></head><body>- <a href="example.com">Link to example</a>+ <a href="https://example.com">Link to example</a><p>{CONTENT}</p>+ <a href="mailto:[email protected]">Email me</a></body></html>"#@@ -1289,10 +1288,10 @@ mod tests {assert_eq!(webpage.title(), Some("Best website".to_string()));assert_eq!(- webpage.links(),+ webpage.anchor_links(),vec![Link {source: "https://www.example.com/whatever".to_string().into(),- destination: "example.com".to_string().into(),+ destination: "https://example.com".to_string().into(),text: "Link to example".to_string()}]);@@ -1939,7 +1938,7 @@ mod tests {<script src="test.com"></script></head><body>- <a href="example.com">Link to example</a>+ <a href="https://example.com">Link to example</a><p>{CONTENT}</p></body></html>@@ -1951,11 +1950,11 @@ mod tests {assert_eq!(webpage.title(), Some("Best website".to_string()));assert_eq!(- webpage.links(),+ webpage.anchor_links(),vec![Link {source: "https://www.example.com/whatever".to_string().into(),- destination: "example.com".to_string().into(),+ destination: "https://example.com".to_string().into(),text: "Link to example".to_string()},// Link {diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rsindex 8bb9b47c..3b0dec3b 100644--- a/core/src/webpage/url.rs+++ b/core/src/webpage/url.rs@@ -39,7 +39,13 @@ impl Display for Url {impl From<String> for Url {fn from(url: String) -> Self {- Url(url)+ let url = if url.ends_with('/') {+ &url[..url.len() - 1]+ } else {+ &url+ };++ Url(url.trim().to_string())}}@@ -161,10 +167,10 @@ impl Url {}pub fn is_full_path(&self) -> bool {- matches!(self.protocol(), "http" | "https" | "pdf")+ matches!(self.protocol(), "http" | "https" | "pdf") || self.0.starts_with("//")}- pub fn prefix_with(&mut self, url: &Url) {+ fn prefix_with(&mut self, url: &Url) {self.0 = match (url.0.ends_with('/'), self.0.starts_with('/')) {(true, true) => {let prot = url.protocol().to_string();@@ -195,6 +201,14 @@ impl Url {}}+ pub fn into_absolute(self, base: &Url) -> Self {+ let mut url = self;+ if !url.is_full_path() {+ url.prefix_with(base);+ }+ url+ }+pub async fn download_bytes(&self, timeout: Duration) -> Option<Vec<u8>> {let client = reqwest::Client::builder().timeout(timeout).build().unwrap();@@ -261,6 +275,14 @@ impl Url {.strip_prefix("://").unwrap_or(&self.0)}++ pub fn path_ends_with(&self, ending: &str) -> bool {+ if self.is_homepage() {+ false+ } else {+ self.without_query().ends_with(ending)+ }+ }}#[cfg(test)]@@ -297,32 +319,50 @@ mod tests {}#[test]- fn prefix_with() {+ fn into_absolute() {let mut a: Url = "/test".to_string().into();let b: Url = "https://example.com".to_string().into();- a.prefix_with(&b);+ a = a.into_absolute(&b);assert_eq!(a.full().as_str(), "https://example.com/test");let mut a: Url = "test".to_string().into();let b: Url = "https://example.com".to_string().into();- a.prefix_with(&b);+ a = a.into_absolute(&b);assert_eq!(a.full().as_str(), "https://example.com/test");let mut a: Url = "test".to_string().into();let b: Url = "https://example.com/".to_string().into();- a.prefix_with(&b);+ a = a.into_absolute(&b);assert_eq!(a.full().as_str(), "https://example.com/test");let mut a: Url = "/test".to_string().into();let b: Url = "https://example.com/".to_string().into();- a.prefix_with(&b);+ a = a.into_absolute(&b);+ assert_eq!(a.full().as_str(), "https://example.com/test");++ let mut a: Url = "https://example.com/test".to_string().into();+ let b: Url = "https://example.com/".to_string().into();+ a = a.into_absolute(&b);+ assert_eq!(a.full().as_str(), "https://example.com/test");++ let mut a: Url = "https://example.com/test".to_string().into();+ let b: Url = "example.com".to_string().into();+ a = a.into_absolute(&b);assert_eq!(a.full().as_str(), "https://example.com/test");++ let mut a: Url = "https://a.com/test".to_string().into();+ let b: Url = "b.com".to_string().into();+ a = a.into_absolute(&b);+ assert_eq!(a.full().as_str(), "https://a.com/test");}#[test]fn is_full_path() {let url: Url = "https://dailymail.co.uk".to_string().into();assert!(url.is_full_path());++ let url: Url = "//dailymail.co.uk".to_string().into();+ assert!(url.is_full_path());}#[test]@@ -391,4 +431,28 @@ mod tests {let url: Url = "podcasts.apple.com".to_string().into();assert!(url.is_homepage());}++ #[test]+ fn path_ends_with() {+ let url: Url = "https://test.example.com/test/test".to_string().into();+ assert!(url.path_ends_with("test"));++ let url: Url = "https://test.example.com/test/test".to_string().into();+ assert!(url.path_ends_with("test/test"));++ let url: Url = "https://test.example.com".to_string().into();+ assert!(!url.path_ends_with("/"));++ let url: Url = "https://test.example.zip".to_string().into();+ assert!(!url.path_ends_with(".zip"));++ let url: Url = "https://test.example.com".to_string().into();+ assert!(!url.path_ends_with(".com"));++ let url: Url = "https://test.example.com/.com".to_string().into();+ assert!(url.path_ends_with(".com"));++ let url: Url = "https://test.example.com/test.png".to_string().into();+ assert!(url.path_ends_with(".png"));+ }}diff --git a/frontend/src/pages/chat.astro b/frontend/src/pages/chat.astroindex 95d3bae9..fb83c3d0 100644--- a/frontend/src/pages/chat.astro+++ b/frontend/src/pages/chat.astro@@ -18,7 +18,7 @@ import OpticSelector from "../components/OpticSelector.astro";<div id="message-container" class="flex px-5 flex-col gap-y-3 h-full"><div class="flex flex-col h-full items-center gap-y-3"><h1 class="text-2xl font-bold">Chat with Alice</h1>- <p class="text-sm">Alice is an AI that tries to answer your questions by searhing for information on the internet. Alice is <b class="font-bold">highly experimental</b> and might produce bad or downright wrong answers. As always, you should verify the information from multiple sources and make sure the sources supports the claims.</p>+ <p class="text-sm">Alice is an AI that tries to answer your questions by searhing for information on the internet. Alice is <b class="font-bold">highly experimental</b> and might produce bad or downright wrong answers. As always, you should verify the information from multiple sources and make sure the sources supports the claims. Alice will most likely be a paid feature in the future.</p></div></div><div class="w-full gap-y-2">diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdxnew file mode 100644index 00000000..813b89b5--- /dev/null+++ b/frontend/src/pages/crawler.mdx@@ -0,0 +1,13 @@+---+layout: "../layouts/Markdown.astro"+title: StractBot+---++# StractBot+StractBot is the name of our crawler that is used to collect pages from the web in order to index them in the search engine. It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler). The crawler uses the user agent `Mozilla/5.0 (compatible; Stractbot/0.1; +https://trystract.com/bot)`.++## Politeness+StractBot is a polite crawler. This means that it respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server. It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 30 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 30 seconds. This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 128).++## Contact us+If you have any concerns or bad experiences with our crawler, please don't hesitate to reach out to us at [[email protected]](mailto:[email protected]).\ No newline at end of file