web search done right
move libtorch behind feature flag
Mikkel Denker276165da4914
diff --git a/configs/api.toml b/configs/api.tomlindex b3113812..16116991 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -1,6 +1,5 @@cluster_id = "dev_api"crossencoder_model_path = "data/cross_encoder"-fact_check_model_path = "data/fact_model"gossip_addr = "0.0.0.0:3005"gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]host = "0.0.0.0:3000"diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex 0a80ea5d..e076dd66 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,5 +1,5 @@host_centrality_store_path = "./data/centrality"-limit_warc_files = 10+limit_warc_files = 20# minimum_clean_words = 40[warc_source]diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 926df329..79a93774 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -10,10 +10,11 @@ version = "0.1.0"[features]cors = []-default = ["cors", "with_alice"]+default = ["cors", "with_alice", "libtorch"]dev = ["cors"]-prod = ["cors"]-with_alice = []+libtorch = ["dep:tch", "dep:torch-sys"]+prod = ["cors", "libtorch"]+with_alice = ["libtorch"][[bin]]name = "stract"@@ -88,13 +89,13 @@ serde = {workspace = true}serde_json = {workspace = true}serde_urlencoded = {workspace = true}tantivy = {workspace = true}-tch = {workspace = true}+tch = {workspace = true, optional = true}thiserror = {workspace = true}tokenizers = {workspace = true}tokio = {workspace = true}tokio-stream = {workspace = true}toml = {workspace = true}-torch-sys = {workspace = true}+torch-sys = {workspace = true, optional = true}tower-http = {workspace = true}tracing = {workspace = true}tracing-subscriber = {workspace = true}@@ -118,6 +119,10 @@ maplit = {workspace = true}proptest = {workspace = true}proptest-derive = {workspace = true}+[[example]]+name = "alice"+required-features = ["with_alice"]+[[bench]]harness = falsename = "search-preindexed"diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rsindex 86b66d57..64e46668 100644--- a/core/benches/search-preindexed.rs+++ b/core/benches/search-preindexed.rs@@ -42,11 +42,14 @@ pub fn criterion_benchmark(c: &mut Criterion) {searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH).into());for _ in 0..1000 {- // bench!("the", searcher, c);- // bench!("dtu", searcher, c);- // bench!("the best", searcher, c);- // bench!("the circle of life", searcher, c);- bench!("runescape is old the", searcher, c);+ bench!("the", searcher, c);+ bench!("dtu", searcher, c);+ bench!("the best", searcher, c);+ bench!("the circle of life", searcher, c);+ bench!("what", searcher, c);+ bench!("a", searcher, c);+ bench!("sun", searcher, c);+ bench!("what a sun", searcher, c);}}diff --git a/core/src/api/alice.rs b/core/src/api/alice.rsdeleted file mode 100644index b4dffd75..00000000--- a/core/src/api/alice.rs+++ /dev/null@@ -1,235 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{convert::Infallible, sync::Arc};--use axum::{- extract,- response::{sse::Event, IntoResponse, Sse},-};-use eventsource_stream::Eventsource;-use http::StatusCode;-use rand::seq::SliceRandom;-use tokio_stream::{Stream, StreamExt};-use utoipa::{IntoParams, ToSchema};--use crate::{- distributed::member::Service,- entrypoint::{self, alice::SaveStateParams},-};--#[derive(Debug, serde::Serialize, serde::Deserialize)]-pub struct SavedState {- pub id: String,- pub uuid: uuid::Uuid,-}--pub async fn save_state(- extract::State(state): extract::State<Arc<super::State>>,- extract::Json(params): extract::Json<SaveStateParams>,-) -> Result<impl IntoResponse, http::StatusCode> {- // We don't use- // #[cfg(not(feature = "with_alice"))]- // {- // return Err(StatusCode::NOT_FOUND);- // }- // here since this would result in a bunch of lint warnings.- // The following is a bit hacky, but works and is easily readable.-- #[cfg(not(feature = "with_alice"))]- let with_alice = false;-- #[cfg(feature = "with_alice")]- let with_alice = true;-- if !with_alice {- return Err(StatusCode::NOT_FOUND);- }-- let client = reqwest::ClientBuilder::default()- .build()- .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;-- let mut members = state.cluster.members().await;-- members.shuffle(&mut rand::thread_rng());-- let (alice_addr, id) = members- .into_iter()- .find_map(|m| {- if let Service::Alice { host } = m.service {- Some((host, m.id))- } else {- None- }- })- .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;-- let alice_url = format!("http://{}/save_state", alice_addr);-- let res = client- .post(alice_url)- .json(¶ms)- .send()- .await- .map_err(|e| {- tracing::error!("Error contacting alice: {}", e);- StatusCode::INTERNAL_SERVER_ERROR- })?;-- let uuid = res.text().await.map_err(|e| {- tracing::error!("Error contacting alice: {}", e);- StatusCode::INTERNAL_SERVER_ERROR- })?;-- let saved_state = SavedState {- id,- uuid: uuid::Uuid::parse_str(&uuid).map_err(|e| {- tracing::error!("Error parsing uuid: {}", e);- StatusCode::INTERNAL_SERVER_ERROR- })?,- };-- Ok(EncodedSavedState::encode(saved_state).0)-}--#[derive(serde::Serialize, serde::Deserialize, Debug, ToSchema)]-pub struct EncodedSavedState(String);--impl EncodedSavedState {- fn encode(saved_state: SavedState) -> Self {- let saved_state = bincode::serialize(&saved_state).unwrap();- let saved_state = base64::encode(saved_state);- Self(saved_state)- }-- fn decode(self) -> Result<SavedState, anyhow::Error> {- let saved_state = base64::decode(self.0)?;- let saved_state = bincode::deserialize(&saved_state)?;- Ok(saved_state)- }-}--#[derive(serde::Serialize, serde::Deserialize, Debug, IntoParams)]-#[serde(rename_all = "camelCase")]-pub struct AliceParams {- message: String,- optic: Option<String>,- prev_state: Option<EncodedSavedState>,-}--#[allow(clippy::unused_async)]-#[utoipa::path(- get,- path = "/beta/api/alice",- params(AliceParams),- responses(- (status = 200, description = "Interact with Alice", body = ExecutionState, content_type = "text/event-stream"),- )-)]-pub async fn alice_route(- extract::State(state): extract::State<Arc<super::State>>,- extract::Query(params): extract::Query<AliceParams>,-) -> std::result::Result<- Sse<impl Stream<Item = std::result::Result<axum::response::sse::Event, Infallible>>>,- StatusCode,-> {- // We don't use- // #[cfg(not(feature = "with_alice"))]- // {- // return Err(StatusCode::NOT_FOUND);- // }- // here since this would result in a bunch of lint warnings.- // The following is a bit hacky, but works and is easily readable.-- #[cfg(not(feature = "with_alice"))]- let with_alice = false;-- #[cfg(feature = "with_alice")]- let with_alice = true;-- if !with_alice {- return Err(StatusCode::NOT_FOUND);- }-- let mut saved_state: Option<SavedState> = None;-- if let Some(prev_state) = params.prev_state {- saved_state = Some(prev_state.decode().map_err(|e| {- tracing::error!("Error decoding saved state: {}", e);- StatusCode::BAD_REQUEST- })?);- }-- let client = reqwest::ClientBuilder::default()- .build()- .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;-- let mut members = state.cluster.members().await;-- members.shuffle(&mut rand::thread_rng());-- let alice_addr = if let Some(id) = saved_state.as_ref().map(|s| s.id.clone()) {- members- .into_iter()- .find_map(|m| {- if m.id == id {- if let Service::Alice { host } = m.service {- Some(host)- } else {- None- }- } else {- None- }- })- .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?- } else {- members- .into_iter()- .find_map(|m| {- if let Service::Alice { host } = m.service {- Some(host)- } else {- None- }- })- .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?- };-- let params = entrypoint::alice::AliceParams {- message: params.message,- prev_state: saved_state.map(|s| s.uuid),- optic: params.optic,- };-- let mut events = client- .get(format!("http://{}", alice_addr))- .query(¶ms)- .send()- .await- .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?- .bytes_stream()- .eventsource();-- let stream = async_stream::stream! {- while let Some(Ok(item)) = events.next().await {- yield Ok(Event::default().data(item.data));- }- };-- Ok(Sse::new(stream))-}diff --git a/core/src/api/docs.rs b/core/src/api/docs.rsindex d5de1841..2984c83b 100644--- a/core/src/api/docs.rs+++ b/core/src/api/docs.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use super::{alice, autosuggest, explore, fact_check, search, sites, summarize, webgraph};+use super::{autosuggest, explore, search, sites, summarize, webgraph};use axum::Router;use utoipa::{Modify, OpenApi};use utoipa_swagger_ui::SwaggerUi;@@ -31,8 +31,6 @@ use utoipa_swagger_ui::SwaggerUi;webgraph::page::outgoing_pages,autosuggest::route,summarize::summarize_route,- fact_check::fact_check_route,- alice::alice_route,sites::sites_export_optic,explore::explore_export_optic,),@@ -81,13 +79,7 @@ use utoipa_swagger_ui::SwaggerUi;crate::entrypoint::webgraph_server::ScoredSite,autosuggest::Suggestion,- fact_check::FactCheckParams,- fact_check::FactCheckResponse,- crate::alice::SimplifiedWebsite,- crate::alice::ExecutionState,- crate::alice::EncodedEncryptedState,- alice::EncodedSavedState,sites::SitesExportOpticParams,explore::ExploreExportOpticParams,@@ -112,22 +104,6 @@ The API might also change quite a bit during the beta period, but we will try toRemember to always give proper attributions to the sources you use from the search results."#.to_string(),);-- #[cfg(not(feature = "with_alice"))]- {- openapi.paths.paths.remove("/beta/api/alice");- openapi.paths.paths.remove("/beta/api/alice/save_state");- openapi.paths.paths.remove("/beta/api/fact_check");-- if let Some(components) = openapi.components.as_mut() {- components.schemas.remove("SimplifiedWebsite");- components.schemas.remove("ExecutionState");- components.schemas.remove("EncodedEncryptedState");- components.schemas.remove("EncodedSavedState");- components.schemas.remove("FactCheckParams");- components.schemas.remove("FactCheckResponse");- }- }}}diff --git a/core/src/api/fact_check.rs b/core/src/api/fact_check.rsdeleted file mode 100644index 3e5e686c..00000000--- a/core/src/api/fact_check.rs+++ /dev/null@@ -1,51 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::sync::Arc;--use axum::{extract, response::IntoResponse, Json};-use utoipa::ToSchema;--#[derive(serde::Deserialize, serde::Serialize, Debug, ToSchema)]-pub struct FactCheckParams {- claim: String,- evidence: String,-}--#[derive(serde::Deserialize, serde::Serialize, Debug, ToSchema)]-pub struct FactCheckResponse {- score: f64,-}--#[utoipa::path(- post,- path = "/beta/api/fact_check",- request_body(content = FactCheckParams),- responses(- (status = 200, description = "Fact check the given claim against the given evidence", body = FactCheckResponse),- )-)]-pub async fn fact_check_route(- extract::State(state): extract::State<Arc<super::State>>,- extract::Json(params): extract::Json<FactCheckParams>,-) -> Result<impl IntoResponse, http::StatusCode> {- let score = state- .fact_checker- .run(¶ms.claim, ¶ms.evidence)- .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?;-- Ok(Json(FactCheckResponse { score }))-}diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex a8c98575..ae343c38 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -29,14 +29,17 @@ use crate::{cluster::Cluster,member::{Member, Service},},- fact_check_model::FactCheckModel,improvement::{store_improvements_loop, ImprovementEvent},leaky_queue::LeakyQueue,- qa_model::QaModel,- ranking::models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},+ ranking::models::lambdamart::LambdaMART,searcher::api::ApiSearcher,- summarizer::Summarizer,};++#[cfg(feature = "libtorch")]+use crate::{+ qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel, summarizer::Summarizer,+};+use anyhow::Result;use std::{net::SocketAddr, sync::Arc};@@ -49,11 +52,9 @@ use axum::{use self::webgraph::RemoteWebgraph;-mod alice;mod autosuggest;mod docs;mod explore;-mod fact_check;pub mod improvement;mod metrics;pub mod search;@@ -69,6 +70,7 @@ pub struct Counters {pub daily_active_users: user_count::UserCount<user_count::Daily>,}+#[cfg(feature = "libtorch")]pub struct State {pub config: ApiConfig,pub searcher: ApiSearcher,@@ -76,7 +78,17 @@ pub struct State {pub autosuggest: Autosuggest,pub counters: Counters,pub summarizer: Arc<Summarizer>,- pub fact_checker: Arc<FactCheckModel>,+ pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,+ pub cluster: Arc<Cluster>,+}++#[cfg(not(feature = "libtorch"))]+pub struct State {+ pub config: ApiConfig,+ pub searcher: ApiSearcher,+ pub remote_webgraph: RemoteWebgraph,+ pub autosuggest: Autosuggest,+ pub counters: Counters,pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,pub cluster: Arc<Cluster>,}@@ -93,22 +105,12 @@ pub async fn favicon() -> impl IntoResponse {pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {let autosuggest = Autosuggest::load_csv(&config.queries_csv_path)?;- let mut cross_encoder = None;-- if let Some(path) = config.crossencoder_model_path.as_ref() {- cross_encoder = Some(CrossEncoderModel::open(path)?);- }let lambda_model = match &config.lambda_model_path {Some(path) => Some(LambdaMART::open(path)?),None => None,};- let qa_model = match &config.qa_model_path {- Some(path) => Some(QaModel::open(path)?),- None => None,- };-let query_store_queue = config.query_store_db_host.clone().map(|db_host| {let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));tokio::spawn(store_improvements_loop(query_store_queue.clone(), db_host));@@ -129,26 +131,55 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {.await?,);let remote_webgraph = RemoteWebgraph::new(cluster.clone());- let searcher = ApiSearcher::new(- cluster.clone(),- cross_encoder,- lambda_model,- qa_model,- bangs,- config.clone(),- );- let state = Arc::new(State {- config: config.clone(),- searcher,- autosuggest,- counters,- remote_webgraph,- summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),- fact_checker: Arc::new(FactCheckModel::open(&config.fact_check_model_path)?),- improvement_queue: query_store_queue,- cluster,- });+ #[cfg(feature = "libtorch")]+ let state = {+ let mut cross_encoder = None;++ if let Some(path) = config.crossencoder_model_path.as_ref() {+ cross_encoder = Some(CrossEncoderModel::open(path)?);+ }++ let qa_model = match &config.qa_model_path {+ Some(path) => Some(QaModel::open(path)?),+ None => None,+ };++ let searcher = ApiSearcher::new(+ cluster.clone(),+ cross_encoder,+ lambda_model,+ qa_model,+ bangs,+ config.clone(),+ );++ Arc::new(State {+ config: config.clone(),+ searcher,+ autosuggest,+ counters,+ remote_webgraph,+ summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),+ improvement_queue: query_store_queue,+ cluster,+ })+ };++ #[cfg(not(feature = "libtorch"))]+ let state = {+ let searcher = ApiSearcher::new(cluster.clone(), lambda_model, bangs, config.clone());++ Arc::new(State {+ config: config.clone(),+ searcher,+ autosuggest,+ counters,+ remote_webgraph,+ improvement_queue: query_store_queue,+ cluster,+ })+ };Ok(Router::new().merge(@@ -190,9 +221,6 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {"/api/webgraph/page/outgoing",post(webgraph::page::outgoing_pages),)- .route("/api/alice", get(alice::alice_route))- .route("/api/alice/save_state", post(alice::save_state))- .route("/api/fact_check", post(fact_check::fact_check_route)).route("/api/sites/export", post(sites::sites_export_optic)).route("/api/explore/export", post(explore::explore_export_optic)).layer(cors_layer()),diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rsindex 0c20a30c..d32a9239 100644--- a/core/src/api/summarize.rs+++ b/core/src/api/summarize.rs@@ -42,6 +42,7 @@ fn summarize_blocking(iter: impl Iterator<Item = String>, tx: UnboundedSender<St}}+#[cfg(feature = "libtorch")]async fn summarize(params: SummarizeParams,state: Arc<State>,@@ -66,6 +67,16 @@ async fn summarize()}+#[cfg(not(feature = "libtorch"))]+async fn summarize(+ params: SummarizeParams,+ state: Arc<State>,+) -> Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>> {+ use futures::stream::BoxStream;++ Err::<Sse<BoxStream<_>>, anyhow::Error>(anyhow::anyhow!("Summarization is not enabled"))+}+#[allow(clippy::unused_async)]#[utoipa::path(get,diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex d128b734..cd80b07c 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -78,7 +78,7 @@ impl Crawler {}pub fn min_crawl_delay_ms() -> u64 {- 1_000+ 5_000}pub fn max_crawl_delay_ms() -> u64 {@@ -89,13 +89,13 @@ impl Crawler {2048.0}- pub fn min_politeness_factor() -> f32 {- 1.0- }-pub fn max_url_slowdown_retry() -> u8 {3}++ pub fn max_redirects() -> usize {+ 5+ }}pub struct WebgraphServer;diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex a653b67c..10f1b162 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -194,7 +194,6 @@ pub struct ApiConfig {pub qa_model_path: Option<String>,pub bangs_path: String,pub summarizer_path: String,- pub fact_check_model_path: String,pub query_store_db_host: Option<String>,pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,@@ -292,12 +291,12 @@ pub struct CrawlerConfig {#[serde(default = "defaults::Crawler::max_politeness_factor")]pub max_politeness_factor: f32,- #[serde(default = "defaults::Crawler::min_politeness_factor")]- pub min_politeness_factor: f32,-#[serde(default = "defaults::Crawler::max_url_slowdown_retry")]pub max_url_slowdown_retry: u8,+ #[serde(default = "defaults::Crawler::max_redirects")]+ pub max_redirects: usize,+pub timeout_seconds: u64,pub s3: S3Config,pub router_hosts: Vec<String>,diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 332524ba..7b7e76f6 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -67,10 +67,6 @@ impl WorkerThread {timeout: Duration,router_hosts: Vec<SocketAddr>,) -> Result<Self> {- if config.politeness_factor < config.min_politeness_factor {- return Err(Error::InvalidPolitenessFactor.into());- }-let mut headers = reqwest::header::HeaderMap::default();headers.insert(reqwest::header::ACCEPT,@@ -86,6 +82,7 @@ impl WorkerThread {.connect_timeout(timeout).http2_keep_alive_interval(None).default_headers(headers)+ .redirect(reqwest::redirect::Policy::limited(config.max_redirects)).user_agent(&config.user_agent.full).build()?;diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 74a0e5a4..230da23a 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.//! The entrypoint module contains all entrypoints that runs the executables.+#[cfg(feature = "with_alice")]pub mod alice;pub mod api;pub mod autosuggest_scrape;diff --git a/core/src/fact_check_model.rs b/core/src/fact_check_model.rsdeleted file mode 100644index e4f68c94..00000000--- a/core/src/fact_check_model.rs+++ /dev/null@@ -1,97 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::path::Path;--use itertools::Itertools;-use tch::Tensor;-use tokenizers::PaddingParams;-use tokenizers::TruncationParams;--const TRUNCATE_INPUT: usize = 512;--#[derive(thiserror::Error, Debug)]-pub enum Error {- #[error("Tokenizer error")]- Tokenizer(#[from] tokenizers::Error),- #[error("Torch error")]- Torch(#[from] tch::TchError),-}--pub struct FactCheckModel {- tokenizer: tokenizers::Tokenizer,- model: tch::CModule,-}--impl FactCheckModel {- pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self, Error> {- let truncation = TruncationParams {- max_length: TRUNCATE_INPUT,- ..Default::default()- };-- let padding = PaddingParams {- ..Default::default()- };-- let mut tokenizer =- tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;-- tokenizer.with_truncation(Some(truncation))?;- tokenizer.with_padding(Some(padding));-- let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;-- Ok(Self { tokenizer, model })- }-- pub fn run(&self, claim: &str, evidence: &str) -> Result<f64, Error> {- let encoded = self.tokenizer.encode((claim, evidence), true)?;-- let ids = encoded.get_ids().iter().map(|i| *i as i64).collect_vec();-- let attention_mask = encoded- .get_attention_mask()- .iter()- .map(|i| *i as i64)- .collect_vec();-- let ids = Tensor::from_slice(&ids).unsqueeze(0);- let attention_mask = Tensor::from_slice(&attention_mask).unsqueeze(0);-- let output = self.model.forward_ts(&[ids, attention_mask])?;-- Ok(output.double_value(&[0, 0]))- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn sanity_check() {- let model =- FactCheckModel::open("../data/fact_model").expect("Failed to find fact-check model");-- let score = model.run(- "Albert Einstein work in the field of computer science",- "Albert Einstein was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time.",- ).unwrap();-- assert!(score > 0.1);- }-}diff --git a/core/src/lib.rs b/core/src/lib.rsindex 681f0f59..ee8fc991 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -31,6 +31,7 @@ mod inverted_index;pub mod mapreduce;+#[cfg(feature = "with_alice")]pub mod alice;mod api;mod autosuggest;@@ -43,7 +44,6 @@ mod distributed;mod entity_index;mod enum_map;mod executor;-mod fact_check_model;mod fastfield_reader;mod human_website_annotations;pub mod hyperloglog;@@ -55,10 +55,12 @@ mod intmap;mod kahan_sum;mod kv;mod leaky_queue;+#[cfg(feature = "libtorch")]mod llm_utils;mod metrics;pub mod naive_bayes;pub mod prehashed;+#[cfg(feature = "libtorch")]mod qa_model;mod query;pub mod ranking;@@ -71,6 +73,7 @@ pub mod similar_sites;mod snippet;pub mod spell;mod subdomain_count;+#[cfg(feature = "libtorch")]pub mod summarizer;mod tokenizer;#[allow(unused)]diff --git a/core/src/main.rs b/core/src/main.rsindex 1a20e327..223ed936 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -46,12 +46,6 @@ struct Args {#[derive(Subcommand)]enum Commands {- /// Commands to deploy the Alice server.- Alice {- #[clap(subcommand)]- options: AliceOptions,- },-/// Build an index.Indexer {#[clap(subcommand)]@@ -132,15 +126,6 @@ enum Crawler {Router { config_path: String },}-/// Commands to deploy Alice.-#[derive(Subcommand)]-enum AliceOptions {- /// Deploy Alice server.- Serve { config_path: String },- /// Generate a new keypair for Alice to sign states.- GenerateKey,-}-/// Commands to train or run inference on the classifier that predicts if a webpage is NSFW or SFW.#[derive(Subcommand)]enum SafetyClassifierOptions {@@ -313,17 +298,6 @@ fn main() -> Result<()> {dmoz_file,output_path,} => entrypoint::dmoz_parser::run(dmoz_file, output_path).unwrap(),- Commands::Alice { options } => match options {- AliceOptions::Serve { config_path } => {- let config: config::AliceLocalConfig = load_toml_config(config_path);-- tokio::runtime::Builder::new_multi_thread()- .enable_all()- .build()?- .block_on(entrypoint::alice::run(config))?- }- AliceOptions::GenerateKey => entrypoint::alice::generate_key(),- },Commands::Crawler { options } => match options {Crawler::Worker { config_path } => {let config: config::CrawlerConfig = load_toml_config(config_path);diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rsindex 29a481e3..46402c29 100644--- a/core/src/ranking/models/cross_encoder.rs+++ b/core/src/ranking/models/cross_encoder.rs@@ -14,50 +14,129 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use anyhow::anyhow;+use anyhow::Result;use std::path::Path;-use itertools::Itertools;-use tch::Tensor;-use tokenizers::PaddingParams;-use tokenizers::TruncationParams;+#[cfg(feature = "libtorch")]+pub use model::*;-const TRUNCATE_INPUT: usize = 128;+#[cfg(feature = "libtorch")]+mod model {+ use super::*;-#[derive(thiserror::Error, Debug)]-pub enum Error {- #[error("Tokenizer error")]- Tokenizer(#[from] tokenizers::Error),- #[error("Torch error")]- Torch(#[from] tch::TchError),-}+ use itertools::Itertools;+ use tch::Tensor;+ use tokenizers::PaddingParams;+ use tokenizers::TruncationParams;-type Result<T> = std::result::Result<T, Error>;+ const TRUNCATE_INPUT: usize = 128;-pub struct CrossEncoderModel {- tokenizer: tokenizers::Tokenizer,- model: tch::CModule,-}+ pub struct CrossEncoderModel {+ tokenizer: tokenizers::Tokenizer,+ model: tch::CModule,+ }-impl CrossEncoderModel {- pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {- let truncation = TruncationParams {- max_length: TRUNCATE_INPUT,- ..Default::default()- };+ impl CrossEncoderModel {+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ let truncation = TruncationParams {+ max_length: TRUNCATE_INPUT,+ ..Default::default()+ };- let padding = PaddingParams {- ..Default::default()- };+ let padding = PaddingParams {+ ..Default::default()+ };- let mut tokenizer =- tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;+ let mut tokenizer =+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))+ .map_err(|_| anyhow!("couldn't open tokenizer"))?;- tokenizer.with_truncation(Some(truncation))?;- tokenizer.with_padding(Some(padding));+ tokenizer+ .with_truncation(Some(truncation))+ .map_err(|_| anyhow!("tokenizer truncation settings"))?;+ tokenizer.with_padding(Some(padding));- let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;+ let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;- Ok(Self { tokenizer, model })+ Ok(Self { tokenizer, model })+ }+ }++ impl CrossEncoder for CrossEncoderModel {+ fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {+ if bodies.is_empty() {+ return Vec::new();+ }++ let bs = bodies.len();++ let input: Vec<_> = bodies+ .iter()+ .map(|body| {+ (+ query.to_string(),+ body.split_whitespace()+ .take(TRUNCATE_INPUT)+ .collect::<String>(),+ )+ })+ .collect();++ let encoded = self.tokenizer.encode_batch(input, true).unwrap();++ let num_tokens = encoded+ .iter()+ .map(|enc| enc.get_ids().len())+ .max()+ .unwrap_or(0);++ let ids = encoded+ .iter()+ .flat_map(|enc| enc.get_ids().iter().map(|i| *i as i64).take(TRUNCATE_INPUT))+ .collect_vec();++ let attention_mask = encoded+ .iter()+ .flat_map(|enc| {+ enc.get_attention_mask()+ .iter()+ .map(|i| *i as i64)+ .take(TRUNCATE_INPUT)+ })+ .collect_vec();++ let type_ids = encoded+ .iter()+ .flat_map(|enc| {+ enc.get_type_ids()+ .iter()+ .map(|i| *i as i64)+ .take(TRUNCATE_INPUT)+ })+ .collect_vec();++ let ids = Tensor::from_slice(&ids).reshape([bs as i64, num_tokens as i64]);+ let attention_mask =+ Tensor::from_slice(&attention_mask).reshape([bs as i64, num_tokens as i64]);+ let type_ids = Tensor::from_slice(&type_ids).reshape([bs as i64, num_tokens as i64]);++ let output = self+ .model+ .forward_ts(&[ids, attention_mask, type_ids])+ .unwrap();++ let mut res = Vec::with_capacity(bs);++ for i in 0..bs {+ let s = output.double_value(&[i as i64, 0]);+ let s = s.exp();+ let s = s / (s + 1.0);+ res.push(s);+ }++ res+ }}}@@ -65,7 +144,7 @@ pub trait CrossEncoder: Send + Sync {fn run(&self, query: &str, bodies: &[String]) -> Vec<f64>;}-pub struct DummyCrossEncoder {}+pub struct DummyCrossEncoder;impl CrossEncoder for DummyCrossEncoder {fn run(&self, _query: &str, bodies: &[String]) -> Vec<f64> {@@ -73,83 +152,8 @@ impl CrossEncoder for DummyCrossEncoder {}}-impl CrossEncoder for CrossEncoderModel {- fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {- if bodies.is_empty() {- return Vec::new();- }-- let bs = bodies.len();-- let input: Vec<_> = bodies- .iter()- .map(|body| {- (- query.to_string(),- body.split_whitespace()- .take(TRUNCATE_INPUT)- .collect::<String>(),- )- })- .collect();-- let encoded = self.tokenizer.encode_batch(input, true).unwrap();-- let num_tokens = encoded- .iter()- .map(|enc| enc.get_ids().len())- .max()- .unwrap_or(0);-- let ids = encoded- .iter()- .flat_map(|enc| enc.get_ids().iter().map(|i| *i as i64).take(TRUNCATE_INPUT))- .collect_vec();-- let attention_mask = encoded- .iter()- .flat_map(|enc| {- enc.get_attention_mask()- .iter()- .map(|i| *i as i64)- .take(TRUNCATE_INPUT)- })- .collect_vec();-- let type_ids = encoded- .iter()- .flat_map(|enc| {- enc.get_type_ids()- .iter()- .map(|i| *i as i64)- .take(TRUNCATE_INPUT)- })- .collect_vec();-- let ids = Tensor::from_slice(&ids).reshape([bs as i64, num_tokens as i64]);- let attention_mask =- Tensor::from_slice(&attention_mask).reshape([bs as i64, num_tokens as i64]);- let type_ids = Tensor::from_slice(&type_ids).reshape([bs as i64, num_tokens as i64]);-- let output = self- .model- .forward_ts(&[ids, attention_mask, type_ids])- .unwrap();-- let mut res = Vec::with_capacity(bs);-- for i in 0..bs {- let s = output.double_value(&[i as i64, 0]);- let s = s.exp();- let s = s / (s + 1.0);- res.push(s);- }-- res- }-}-#[cfg(test)]+#[cfg(feature = "libtorch")]mod tests {use super::*;diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 6caa4968..cb2336eb 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -28,13 +28,12 @@ use crate::{};use super::{- models::{- cross_encoder::CrossEncoder,- lambdamart::{self, LambdaMART},- },+ models::lambdamart::{self, LambdaMART},Signal, SignalAggregator, SignalCoefficient, SignalScore,};+use super::models::cross_encoder::CrossEncoder;+pub trait AsRankingWebsite: Clone {fn as_ranking(&self) -> &RankingWebsite;fn as_mut_ranking(&mut self) -> &mut RankingWebsite;diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex 91493f5c..60d0118e 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -27,6 +27,8 @@ use url::Url;use crate::bangs::{Bang, BangHit};use crate::config::{ApiConfig, ApiThresholds, CollectorConfig};use crate::inverted_index::RetrievedWebpage;+#[cfg(not(feature = "libtorch"))]+use crate::ranking::models::cross_encoder::DummyCrossEncoder;use crate::ranking::ALL_SIGNALS;use crate::search_prettifier::{create_stackoverflow_sidebar, DisplayedAnswer, DisplayedEntity, DisplayedSidebar,@@ -37,19 +39,18 @@ use crate::{bangs::Bangs,collector::BucketCollector,distributed::cluster::Cluster,- qa_model::QaModel,- ranking::{- models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},- pipeline::RankingPipeline,- },+ ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},};use crate::{ceil_char_boundary, floor_char_boundary, query, Result};+#[cfg(feature = "libtorch")]+use crate::{qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel};use super::{distributed, DistributedSearcher, InitialSearchResultShard, ScoredWebsitePointer, SearchQuery,SearchResult, WebsitesResult,};+#[cfg(feature = "libtorch")]pub struct ApiSearcher {distributed_searcher: DistributedSearcher,cross_encoder: Option<Arc<CrossEncoderModel>>,@@ -61,7 +62,18 @@ pub struct ApiSearcher {widgets: Widgets,}+#[cfg(not(feature = "libtorch"))]+pub struct ApiSearcher {+ distributed_searcher: DistributedSearcher,+ lambda_model: Option<Arc<LambdaMART>>,+ bangs: Bangs,+ collector_config: CollectorConfig,+ thresholds: ApiThresholds,+ widgets: Widgets,+}+impl ApiSearcher {+ #[cfg(feature = "libtorch")]pub fn new(cluster: Arc<Cluster>,cross_encoder: Option<CrossEncoderModel>,@@ -82,6 +94,23 @@ impl ApiSearcher {}}+ #[cfg(not(feature = "libtorch"))]+ pub fn new(+ cluster: Arc<Cluster>,+ lambda_model: Option<LambdaMART>,+ bangs: Bangs,+ config: ApiConfig,+ ) -> Self {+ Self {+ distributed_searcher: DistributedSearcher::new(cluster),+ lambda_model: lambda_model.map(Arc::new),+ bangs,+ collector_config: config.collector,+ thresholds: config.thresholds,+ widgets: Widgets::new(config.widgets).unwrap(),+ }+ }+fn combine_results(&self,initial_results: Vec<InitialSearchResultShard>,@@ -241,6 +270,7 @@ impl ApiSearcher {..Default::default()};+ #[cfg(feature = "libtorch")]let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::reranking_for_query(&mut query,self.cross_encoder.as_ref().map(Arc::clone),@@ -248,6 +278,15 @@ impl ApiSearcher {self.collector_config.clone(),)?;+ #[cfg(not(feature = "libtorch"))]+ let pipeline: RankingPipeline<ScoredWebsitePointer> =+ RankingPipeline::reranking_for_query::<DummyCrossEncoder>(+ &mut query,+ None,+ self.lambda_model.clone(),+ self.collector_config.clone(),+ )?;+let initial_results = self.distributed_searcher.search_initial(&query).await;if initial_results.is_empty() {@@ -321,6 +360,7 @@ impl ApiSearcher {}let mut search_query = query.clone();+ #[cfg(feature = "libtorch")]let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::reranking_for_query(&mut search_query,self.cross_encoder.as_ref().map(Arc::clone),@@ -328,6 +368,15 @@ impl ApiSearcher {self.collector_config.clone(),)?;+ #[cfg(not(feature = "libtorch"))]+ let pipeline: RankingPipeline<ScoredWebsitePointer> =+ RankingPipeline::reranking_for_query::<DummyCrossEncoder>(+ &mut search_query,+ None,+ self.lambda_model.clone(),+ self.collector_config.clone(),+ )?;+let initial_results = self.distributed_searcher.search_initial(&search_query)@@ -419,6 +468,7 @@ impl ApiSearcher {)}+ #[cfg(feature = "libtorch")]fn answer(&self, query: &str, webpages: &mut Vec<DisplayedWebpage>) -> Option<DisplayedAnswer> {self.qa_model.as_ref().and_then(|qa_model| {let contexts: Vec<_> = webpages@@ -452,6 +502,11 @@ impl ApiSearcher {})}+ #[cfg(not(feature = "libtorch"))]+ fn answer(&self, query: &str, webpages: &mut Vec<DisplayedWebpage>) -> Option<DisplayedAnswer> {+ None+ }+pub async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {self.distributed_searcher.get_webpage(url).await}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex a7977af0..e370c08c 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -27,6 +27,8 @@ use crate::index::Index;use crate::inverted_index::RetrievedWebpage;use crate::query::Query;use crate::ranking::centrality_store::SearchCentralityStore;+#[cfg(not(feature = "libtorch"))]+use crate::ranking::models::cross_encoder::DummyCrossEncoder;use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};@@ -67,7 +69,7 @@ struct InvertedIndexResult {impl LocalSearcher {pub fn new(index: Index) -> Self {let mut index = index;- index.optimize_for_search().unwrap();+ // index.optimize_for_search().unwrap();LocalSearcher {index,@@ -315,26 +317,39 @@ impl LocalSearcher {pub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {use std::time::Instant;- use crate::{- ranking::models::cross_encoder::CrossEncoderModel, search_prettifier::DisplayedSidebar,- };+ use crate::search_prettifier::DisplayedSidebar;let start = Instant::now();let mut search_query = query.clone();- let pipeline = match CrossEncoderModel::open("data/cross_encoder") {- Ok(model) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(- &mut search_query,- Some(Arc::new(model)),- None,- self.collector_config.clone(),- )?,- Err(_) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(+ #[cfg(feature = "libtorch")]+ let pipeline = {+ use crate::ranking::models::cross_encoder::CrossEncoderModel;+ match CrossEncoderModel::open("data/cross_encoder") {+ Ok(model) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(+ &mut search_query,+ Some(Arc::new(model)),+ None,+ self.collector_config.clone(),+ )?,+ Err(_) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(+ &mut search_query,+ None,+ None,+ self.collector_config.clone(),+ )?,+ }+ };++ #[cfg(not(feature = "libtorch"))]+ let pipeline = {+ use crate::ranking::models::cross_encoder::DummyCrossEncoder;+ RankingPipeline::reranking_for_query::<DummyCrossEncoder>(&mut search_query,None,None,self.collector_config.clone(),- )?,+ )?};let search_result = self.search_initial(&search_query, true)?;diff --git a/frontend/src/routes/webmasters/README.md b/frontend/src/routes/webmasters/README.mdindex dfb831a4..ccd562ac 100644--- a/frontend/src/routes/webmasters/README.md+++ b/frontend/src/routes/webmasters/README.md@@ -7,7 +7,7 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open so## PolitenessStractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime \* politeness, 1 sec), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 1 sec between requests and at most 60 seconds.+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime \* politeness, 5 sec), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 5 sec between requests and at most 60 seconds.This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.diff --git a/justfile b/justfileindex 296a3af4..77b67785 100644--- a/justfile+++ b/justfile@@ -21,8 +21,11 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")cd frontend && npm run openapi@setup *ARGS:- just setup_python_env+ python3 -m venv .venv || truejust download_libtorch {{ARGS}}++@prepare_models:+ just setup_python_env./scripts/export_crossencoder./scripts/export_qa_model./scripts/export_abstractive_summary_model@@ -31,6 +34,7 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")@configure *ARGS:just setup {{ARGS}}+ just prepare_modelsRUST_LOG="none,stract=info" just cargo run --release --all-features -- configure {{ARGS}}@setup_python_env: