web search done right
Rename Rust `frontend` to `api` (#86)
Oliver Bøvingc7e941f3c4d5
diff --git a/configs/frontend.toml b/configs/api.tomlsimilarity index 90%rename from configs/frontend.tomlrename to configs/api.tomlindex 0e6b6dfc..610e979a 100644--- a/configs/frontend.toml+++ b/configs/api.toml@@ -1,4 +1,4 @@-cluster_id = "dev_frontend"+cluster_id = "dev_api"queries_csv_path = "data/queries_us.csv"host = "0.0.0.0:3000"prometheus_host = "0.0.0.0:3001"@@ -13,4 +13,4 @@ summarizer_path = "data/summarizer"[thresholds]stackoverflow = 0.0-entity_sidebar = 0.0\ No newline at end of file+entity_sidebar = 0.0diff --git a/core/askama.toml b/core/askama.tomldeleted file mode 100644index 20032bdb..00000000--- a/core/askama.toml+++ /dev/null@@ -1,5 +0,0 @@-[general]-# Directories to search for templates, relative to the crate root.-dirs = ["../frontend/dist/"]-# Unless you add a `-` in a block, whitespace characters won't be trimmed.-whitespace = "preserve"diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex b261fb69..d484a374 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -14,10 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-//! The api module contains the http api and frontend.+//! The api module contains the http api.//! All http requests are handled using axum.-//! The frontend is served using a combination of axum, askama and astro-//! (with some funky astro hacks to make askama and astro play nice together).use axum::{body::Body, extract, middleware, Router};use tokio::sync::Mutex;@@ -26,7 +24,7 @@ use tower_http::compression::CompressionLayer;use crate::{autosuggest::Autosuggest,bangs::Bangs,- config::FrontendConfig,+ config::ApiConfig,distributed::{cluster::Cluster,member::{Member, Service},@@ -36,7 +34,7 @@ use crate::{leaky_queue::LeakyQueue,qa_model::QaModel,ranking::models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},- searcher::frontend::FrontendSearcher,+ searcher::api::ApiSearcher,summarizer::Summarizer,};use anyhow::Result;@@ -70,8 +68,8 @@ pub struct Counters {}pub struct State {- pub config: FrontendConfig,- pub searcher: FrontendSearcher,+ pub config: ApiConfig,+ pub searcher: ApiSearcher,pub remote_webgraph: RemoteWebgraph,pub autosuggest: Autosuggest,pub counters: Counters,@@ -91,7 +89,7 @@ pub async fn favicon() -> impl IntoResponse {.unwrap()}-pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Router> {+pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {let autosuggest = Autosuggest::load_csv(&config.queries_csv_path)?;let mut cross_encoder = None;@@ -121,7 +119,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<RouteCluster::join(Member {id: config.cluster_id.clone(),- service: Service::Frontend { host: config.host },+ service: Service::Api { host: config.host },},config.gossip_addr,config.gossip_seed_nodes.clone().unwrap_or_default(),@@ -129,7 +127,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route.await?,);let remote_webgraph = RemoteWebgraph::new(cluster.clone());- let searcher = FrontendSearcher::new(+ let searcher = ApiSearcher::new(cluster.clone(),cross_encoder,lambda_model,diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex 2447a569..9ad0ae60 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -34,9 +34,9 @@ impl Collector {}}-pub struct Frontend;+pub struct Api;-impl Frontend {+impl Api {pub fn stackoverflow() -> f64 {0.5}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex acaff635..961ddf82 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -171,29 +171,29 @@ impl Default for CollectorConfig {}#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct FrontendThresholds {- #[serde(default = "defaults::Frontend::stackoverflow")]+pub struct ApiThresholds {+ #[serde(default = "defaults::Api::stackoverflow")]pub stackoverflow: f64,- #[serde(default = "defaults::Frontend::entity_sidebar")]+ #[serde(default = "defaults::Api::entity_sidebar")]pub entity_sidebar: f64,- #[serde(default = "defaults::Frontend::discussions_widget")]+ #[serde(default = "defaults::Api::discussions_widget")]pub discussions_widget: f64,}-impl Default for FrontendThresholds {+impl Default for ApiThresholds {fn default() -> Self {Self {- stackoverflow: defaults::Frontend::stackoverflow(),- entity_sidebar: defaults::Frontend::entity_sidebar(),- discussions_widget: defaults::Frontend::discussions_widget(),+ stackoverflow: defaults::Api::stackoverflow(),+ entity_sidebar: defaults::Api::entity_sidebar(),+ discussions_widget: defaults::Api::discussions_widget(),}}}#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct FrontendConfig {+pub struct ApiConfig {pub queries_csv_path: String,pub host: SocketAddr,pub prometheus_host: SocketAddr,@@ -213,7 +213,7 @@ pub struct FrontendConfig {pub collector: CollectorConfig,#[serde(default)]- pub thresholds: FrontendThresholds,+ pub thresholds: ApiThresholds,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rsindex 42c1b5d4..d38c75f8 100644--- a/core/src/distributed/member.rs+++ b/core/src/distributed/member.rs@@ -23,7 +23,7 @@ use crate::searcher::ShardId;#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]pub enum Service {Searcher { host: SocketAddr, shard: ShardId },- Frontend { host: SocketAddr },+ Api { host: SocketAddr },Webgraph { host: SocketAddr },Alice { host: SocketAddr },}diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rsindex 341cec34..968a1949 100644--- a/core/src/entrypoint/alice.rs+++ b/core/src/entrypoint/alice.rs@@ -115,14 +115,14 @@ pub async fn route(.await.into_iter().find_map(|m| {- if let Service::Frontend { host } = m.service {+ if let Service::Api { host } = m.service {Some(host)} else {None}}).ok_or_else(|| {- info!("no frontend found");+ info!("no api found");http::StatusCode::INTERNAL_SERVER_ERROR})?;diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/api.rssimilarity index 95%rename from core/src/entrypoint/frontend.rsrename to core/src/entrypoint/api.rsindex 90aac536..e0c03864 100644--- a/core/src/entrypoint/frontend.rs+++ b/core/src/entrypoint/api.rs@@ -22,7 +22,7 @@ use crate::{metrics::Label,};-pub async fn run(config: config::FrontendConfig) -> Result<()> {+pub async fn run(config: config::ApiConfig) -> Result<()> {let search_counter_success = crate::metrics::Counter::default();let search_counter_fail = crate::metrics::Counter::default();let explore_counter = crate::metrics::Counter::default();@@ -69,7 +69,7 @@ pub async fn run(config: config::FrontendConfig) -> Result<()> {let metrics_app = metrics_router(registry);let addr = config.host;- tracing::info!("frontend server listening on {}", addr);+ tracing::info!("api server listening on {}", addr);let server = axum::Server::bind(&addr).serve(app.into_make_service());let addr = config.prometheus_host;diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 1b3a8308..74a0e5a4 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -16,6 +16,7 @@//! The entrypoint module contains all entrypoints that runs the executables.pub mod alice;+pub mod api;pub mod autosuggest_scrape;mod centrality;#[cfg(feature = "dev")]@@ -23,7 +24,6 @@ pub mod configure;pub mod crawler;pub mod dmoz_parser;mod entity;-pub mod frontend;pub mod indexer;pub mod safety_classifier;pub mod search_server;diff --git a/core/src/main.rs b/core/src/main.rsindex cc861717..48b4a441 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -25,7 +25,7 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};use stract::entrypoint::configure;use stract::entrypoint::indexer::IndexPointer;-use stract::entrypoint::{self, frontend, safety_classifier, search_server, webgraph_server};+use stract::entrypoint::{self, api, safety_classifier, search_server, webgraph_server};use stract::webgraph::WebgraphBuilder;use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -83,9 +83,9 @@ enum Commands {/// Deploy the search server.SearchServer { config_path: String },- /// Deploy the frontend. The frontend is a web server that serves the search UI and interacts with+ /// Deploy the API. The API is a web server that serves the search UI and interacts with/// the search servers, webgraph servers etc. to provide the necesarry functionality.- Frontend { config_path: String },+ Api { config_path: String },/// Scrape the Google autosuggest API for search queries.AutosuggestScrape {@@ -264,13 +264,13 @@ fn main() -> Result<()> {.block_on(webgraph_server::run(config))?}},- Commands::Frontend { config_path } => {- let config: config::FrontendConfig = load_toml_config(config_path);+ Commands::Api { config_path } => {+ let config: config::ApiConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(frontend::run(config))?+ .block_on(api::run(config))?}Commands::SearchServer { config_path } => {let config: config::SearchServerConfig = load_toml_config(config_path);diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/api.rssimilarity index 97%rename from core/src/searcher/frontend.rsrename to core/src/searcher/api.rsindex caa5e81a..94734406 100644--- a/core/src/searcher/frontend.rs+++ b/core/src/searcher/api.rs@@ -25,7 +25,7 @@ use optics::Optic;use url::Url;use crate::bangs::{Bang, BangHit};-use crate::config::{CollectorConfig, FrontendThresholds};+use crate::config::{ApiThresholds, CollectorConfig};use crate::inverted_index::RetrievedWebpage;use crate::ranking::ALL_SIGNALS;use crate::search_prettifier::{@@ -50,17 +50,17 @@ use super::{SearchResult, WebsitesResult,};-pub struct FrontendSearcher {+pub struct ApiSearcher {distributed_searcher: DistributedSearcher,cross_encoder: Option<Arc<CrossEncoderModel>>,lambda_model: Option<Arc<LambdaMART>>,qa_model: Option<Arc<QaModel>>,bangs: Bangs,collector_config: CollectorConfig,- thresholds: FrontendThresholds,+ thresholds: ApiThresholds,}-impl FrontendSearcher {+impl ApiSearcher {pub fn new(cluster: Arc<Cluster>,cross_encoder: Option<CrossEncoderModel>,@@ -68,7 +68,7 @@ impl FrontendSearcher {qa_model: Option<QaModel>,bangs: Bangs,collector_config: CollectorConfig,- thresholds: FrontendThresholds,+ thresholds: ApiThresholds,) -> Self {Self {distributed_searcher: DistributedSearcher::new(cluster),@@ -519,22 +519,22 @@ mod tests {"this is a test".to_string());let input = r#"-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test"#;let res = generate_answer_snippet(input, 0..500);diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex cc820982..db572d80 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -14,8 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod api;pub mod distributed;-pub mod frontend;pub mod local;pub use distributed::*;diff --git a/scripts/run_frontend.py b/scripts/run_frontend.pyindex 1351dab7..a06ea5b3 100755--- a/scripts/run_frontend.py+++ b/scripts/run_frontend.py@@ -21,7 +21,7 @@ processes = []processes.append(subprocess.Popen(- ["cargo", "run"] + release + ["--", "frontend", "configs/frontend.toml"]+ ["cargo", "run"] + release + ["--", "api", "configs/api.toml"]))processes.append(