Skip to content

web search done right

Rename Rust `frontend` to `api` (#86)

Oliver Bøvingc7e941f3c4d5
diff --git a/configs/frontend.toml b/configs/api.toml
similarity index 90%
rename from configs/frontend.toml
rename to configs/api.toml
index 0e6b6dfc..610e979a 100644
--- a/configs/frontend.toml
+++ b/configs/api.toml
@@ -1,4 +1,4 @@
-cluster_id = "dev_frontend"
+cluster_id = "dev_api"
queries_csv_path = "data/queries_us.csv"
host = "0.0.0.0:3000"
prometheus_host = "0.0.0.0:3001"
@@ -13,4 +13,4 @@ summarizer_path = "data/summarizer"
[thresholds]
stackoverflow = 0.0
-entity_sidebar = 0.0
\ No newline at end of file
+entity_sidebar = 0.0
diff --git a/core/askama.toml b/core/askama.toml
deleted file mode 100644
index 20032bdb..00000000
--- a/core/askama.toml
+++ /dev/null
@@ -1,5 +0,0 @@
-[general]
-# Directories to search for templates, relative to the crate root.
-dirs = ["../frontend/dist/"]
-# Unless you add a `-` in a block, whitespace characters won't be trimmed.
-whitespace = "preserve"
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index b261fb69..d484a374 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -14,10 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-//! The api module contains the http api and frontend.
+//! The api module contains the http api.
//! All http requests are handled using axum.
-//! The frontend is served using a combination of axum, askama and astro
-//! (with some funky astro hacks to make askama and astro play nice together).
use axum::{body::Body, extract, middleware, Router};
use tokio::sync::Mutex;
@@ -26,7 +24,7 @@ use tower_http::compression::CompressionLayer;
use crate::{
autosuggest::Autosuggest,
bangs::Bangs,
- config::FrontendConfig,
+ config::ApiConfig,
distributed::{
cluster::Cluster,
member::{Member, Service},
@@ -36,7 +34,7 @@ use crate::{
leaky_queue::LeakyQueue,
qa_model::QaModel,
ranking::models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},
- searcher::frontend::FrontendSearcher,
+ searcher::api::ApiSearcher,
summarizer::Summarizer,
};
use anyhow::Result;
@@ -70,8 +68,8 @@ pub struct Counters {
}
pub struct State {
- pub config: FrontendConfig,
- pub searcher: FrontendSearcher,
+ pub config: ApiConfig,
+ pub searcher: ApiSearcher,
pub remote_webgraph: RemoteWebgraph,
pub autosuggest: Autosuggest,
pub counters: Counters,
@@ -91,7 +89,7 @@ pub async fn favicon() -> impl IntoResponse {
.unwrap()
}
-pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Router> {
+pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
let autosuggest = Autosuggest::load_csv(&config.queries_csv_path)?;
let mut cross_encoder = None;
@@ -121,7 +119,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route
Cluster::join(
Member {
id: config.cluster_id.clone(),
- service: Service::Frontend { host: config.host },
+ service: Service::Api { host: config.host },
},
config.gossip_addr,
config.gossip_seed_nodes.clone().unwrap_or_default(),
@@ -129,7 +127,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route
.await?,
);
let remote_webgraph = RemoteWebgraph::new(cluster.clone());
- let searcher = FrontendSearcher::new(
+ let searcher = ApiSearcher::new(
cluster.clone(),
cross_encoder,
lambda_model,
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index 2447a569..9ad0ae60 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -34,9 +34,9 @@ impl Collector {
}
}
-pub struct Frontend;
+pub struct Api;
-impl Frontend {
+impl Api {
pub fn stackoverflow() -> f64 {
0.5
}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index acaff635..961ddf82 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -171,29 +171,29 @@ impl Default for CollectorConfig {
}
#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct FrontendThresholds {
- #[serde(default = "defaults::Frontend::stackoverflow")]
+pub struct ApiThresholds {
+ #[serde(default = "defaults::Api::stackoverflow")]
pub stackoverflow: f64,
- #[serde(default = "defaults::Frontend::entity_sidebar")]
+ #[serde(default = "defaults::Api::entity_sidebar")]
pub entity_sidebar: f64,
- #[serde(default = "defaults::Frontend::discussions_widget")]
+ #[serde(default = "defaults::Api::discussions_widget")]
pub discussions_widget: f64,
}
-impl Default for FrontendThresholds {
+impl Default for ApiThresholds {
fn default() -> Self {
Self {
- stackoverflow: defaults::Frontend::stackoverflow(),
- entity_sidebar: defaults::Frontend::entity_sidebar(),
- discussions_widget: defaults::Frontend::discussions_widget(),
+ stackoverflow: defaults::Api::stackoverflow(),
+ entity_sidebar: defaults::Api::entity_sidebar(),
+ discussions_widget: defaults::Api::discussions_widget(),
}
}
}
#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct FrontendConfig {
+pub struct ApiConfig {
pub queries_csv_path: String,
pub host: SocketAddr,
pub prometheus_host: SocketAddr,
@@ -213,7 +213,7 @@ pub struct FrontendConfig {
pub collector: CollectorConfig,
#[serde(default)]
- pub thresholds: FrontendThresholds,
+ pub thresholds: ApiThresholds,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rs
index 42c1b5d4..d38c75f8 100644
--- a/core/src/distributed/member.rs
+++ b/core/src/distributed/member.rs
@@ -23,7 +23,7 @@ use crate::searcher::ShardId;
#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
pub enum Service {
Searcher { host: SocketAddr, shard: ShardId },
- Frontend { host: SocketAddr },
+ Api { host: SocketAddr },
Webgraph { host: SocketAddr },
Alice { host: SocketAddr },
}
diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rs
index 341cec34..968a1949 100644
--- a/core/src/entrypoint/alice.rs
+++ b/core/src/entrypoint/alice.rs
@@ -115,14 +115,14 @@ pub async fn route(
.await
.into_iter()
.find_map(|m| {
- if let Service::Frontend { host } = m.service {
+ if let Service::Api { host } = m.service {
Some(host)
} else {
None
}
})
.ok_or_else(|| {
- info!("no frontend found");
+ info!("no api found");
http::StatusCode::INTERNAL_SERVER_ERROR
})?;
diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/api.rs
similarity index 95%
rename from core/src/entrypoint/frontend.rs
rename to core/src/entrypoint/api.rs
index 90aac536..e0c03864 100644
--- a/core/src/entrypoint/frontend.rs
+++ b/core/src/entrypoint/api.rs
@@ -22,7 +22,7 @@ use crate::{
metrics::Label,
};
-pub async fn run(config: config::FrontendConfig) -> Result<()> {
+pub async fn run(config: config::ApiConfig) -> Result<()> {
let search_counter_success = crate::metrics::Counter::default();
let search_counter_fail = crate::metrics::Counter::default();
let explore_counter = crate::metrics::Counter::default();
@@ -69,7 +69,7 @@ pub async fn run(config: config::FrontendConfig) -> Result<()> {
let metrics_app = metrics_router(registry);
let addr = config.host;
- tracing::info!("frontend server listening on {}", addr);
+ tracing::info!("api server listening on {}", addr);
let server = axum::Server::bind(&addr).serve(app.into_make_service());
let addr = config.prometheus_host;
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index 1b3a8308..74a0e5a4 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -16,6 +16,7 @@
//! The entrypoint module contains all entrypoints that runs the executables.
pub mod alice;
+pub mod api;
pub mod autosuggest_scrape;
mod centrality;
#[cfg(feature = "dev")]
@@ -23,7 +24,6 @@ pub mod configure;
pub mod crawler;
pub mod dmoz_parser;
mod entity;
-pub mod frontend;
pub mod indexer;
pub mod safety_classifier;
pub mod search_server;
diff --git a/core/src/main.rs b/core/src/main.rs
index cc861717..48b4a441 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -25,7 +25,7 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};
use stract::entrypoint::configure;
use stract::entrypoint::indexer::IndexPointer;
-use stract::entrypoint::{self, frontend, safety_classifier, search_server, webgraph_server};
+use stract::entrypoint::{self, api, safety_classifier, search_server, webgraph_server};
use stract::webgraph::WebgraphBuilder;
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -83,9 +83,9 @@ enum Commands {
/// Deploy the search server.
SearchServer { config_path: String },
- /// Deploy the frontend. The frontend is a web server that serves the search UI and interacts with
+ /// Deploy the API. The API is a web server that serves the search UI and interacts with
/// the search servers, webgraph servers etc. to provide the necesarry functionality.
- Frontend { config_path: String },
+ Api { config_path: String },
/// Scrape the Google autosuggest API for search queries.
AutosuggestScrape {
@@ -264,13 +264,13 @@ fn main() -> Result<()> {
.block_on(webgraph_server::run(config))?
}
},
- Commands::Frontend { config_path } => {
- let config: config::FrontendConfig = load_toml_config(config_path);
+ Commands::Api { config_path } => {
+ let config: config::ApiConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
- .block_on(frontend::run(config))?
+ .block_on(api::run(config))?
}
Commands::SearchServer { config_path } => {
let config: config::SearchServerConfig = load_toml_config(config_path);
diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/api.rs
similarity index 97%
rename from core/src/searcher/frontend.rs
rename to core/src/searcher/api.rs
index caa5e81a..94734406 100644
--- a/core/src/searcher/frontend.rs
+++ b/core/src/searcher/api.rs
@@ -25,7 +25,7 @@ use optics::Optic;
use url::Url;
use crate::bangs::{Bang, BangHit};
-use crate::config::{CollectorConfig, FrontendThresholds};
+use crate::config::{ApiThresholds, CollectorConfig};
use crate::inverted_index::RetrievedWebpage;
use crate::ranking::ALL_SIGNALS;
use crate::search_prettifier::{
@@ -50,17 +50,17 @@ use super::{
SearchResult, WebsitesResult,
};
-pub struct FrontendSearcher {
+pub struct ApiSearcher {
distributed_searcher: DistributedSearcher,
cross_encoder: Option<Arc<CrossEncoderModel>>,
lambda_model: Option<Arc<LambdaMART>>,
qa_model: Option<Arc<QaModel>>,
bangs: Bangs,
collector_config: CollectorConfig,
- thresholds: FrontendThresholds,
+ thresholds: ApiThresholds,
}
-impl FrontendSearcher {
+impl ApiSearcher {
pub fn new(
cluster: Arc<Cluster>,
cross_encoder: Option<CrossEncoderModel>,
@@ -68,7 +68,7 @@ impl FrontendSearcher {
qa_model: Option<QaModel>,
bangs: Bangs,
collector_config: CollectorConfig,
- thresholds: FrontendThresholds,
+ thresholds: ApiThresholds,
) -> Self {
Self {
distributed_searcher: DistributedSearcher::new(cluster),
@@ -519,22 +519,22 @@ mod tests {
"this is a test".to_string()
);
let input = r#"
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
"#;
let res = generate_answer_snippet(input, 0..500);
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index cc820982..db572d80 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -14,8 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub mod api;
pub mod distributed;
-pub mod frontend;
pub mod local;
pub use distributed::*;
diff --git a/scripts/run_frontend.py b/scripts/run_frontend.py
index 1351dab7..a06ea5b3 100755
--- a/scripts/run_frontend.py
+++ b/scripts/run_frontend.py
@@ -21,7 +21,7 @@ processes = []
processes.append(
subprocess.Popen(
- ["cargo", "run"] + release + ["--", "frontend", "configs/frontend.toml"]
+ ["cargo", "run"] + release + ["--", "api", "configs/api.toml"]
)
)
processes.append(