web search done right
Add more endpoints to the OpenAPI specification (#80)
Oliver Bøvingf0f0c420bbc8
diff --git a/Cargo.toml b/Cargo.tomlindex d7bea3a8..93b0dedb 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -56,7 +56,7 @@ memmap = "0.7.0"once_cell = "1.13.1"regex = "1.6.0"http = "0.2.8"-tower-http = { version = "0.4.0", features = ["compression-gzip"] }+tower-http = { version = "0.4.0", features = ["compression-gzip", "cors"] }lalrpop-util = { version = "0.19.8", features = ["lexer"] }lalrpop = { version = "0.19.8", features = ["lexer"] }serde_urlencoded = "0.7.1"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 680c9a85..9faccb4f 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -9,7 +9,9 @@ default-run = "stract"# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[features]-dev = []+default = ["cors"]+dev = ["cors"]+cors = [][[bin]]name = "stract"@@ -147,4 +149,4 @@ harness = false[[bench]]name = "naive-bayes"-harness = false\ No newline at end of file+harness = falsediff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex 568c7cd4..73e13214 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -40,6 +40,7 @@ use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};use half::bf16;use tch::Tensor;use url::Url;+use utoipa::ToSchema;use crate::{api::search::ApiSearchQuery,@@ -90,7 +91,7 @@ pub enum Error {LastMessageNotUser,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, ToSchema)]pub struct SimplifiedWebsite {pub title: String,pub text: String,@@ -112,7 +113,7 @@ impl SimplifiedWebsite {}}-#[derive(Debug, serde::Serialize, serde::Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]#[serde(tag = "type", rename_all = "camelCase")]pub enum ExecutionState {BeginSearch {@@ -402,7 +403,7 @@ impl Tokenizer {}}-#[derive(Debug, serde::Serialize, serde::Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]/// base64 encoded `EncryptedState`pub struct EncodedEncryptedState(String);diff --git a/core/src/api/alice.rs b/core/src/api/alice.rsindex 7d8dbe62..59579e38 100644--- a/core/src/api/alice.rs+++ b/core/src/api/alice.rs@@ -24,6 +24,7 @@ use eventsource_stream::Eventsource;use http::StatusCode;use rand::seq::SliceRandom;use tokio_stream::{Stream, StreamExt};+use utoipa::{IntoParams, ToSchema};use crate::{distributed::member::Service,@@ -91,7 +92,7 @@ pub async fn save_state(Ok(EncodedSavedState::encode(saved_state).0)}-#[derive(serde::Serialize, serde::Deserialize, Debug)]+#[derive(serde::Serialize, serde::Deserialize, Debug, ToSchema)]pub struct EncodedSavedState(String);impl EncodedSavedState {@@ -108,17 +109,26 @@ impl EncodedSavedState {}}-#[derive(serde::Serialize, serde::Deserialize, Debug)]+#[derive(serde::Serialize, serde::Deserialize, Debug, IntoParams)]#[serde(rename_all = "camelCase")]-pub struct Params {+pub struct AliceParams {message: String,optic: Option<String>,prev_state: Option<EncodedSavedState>,}-pub async fn route(+#[allow(clippy::unused_async)]+#[utoipa::path(+ get,+ path = "/beta/api/alice",+ params(AliceParams),+ responses(+ (status = 200, description = "Interact with Alice", body = ExecutionState),+ )+)]+pub async fn alice_route(extract::State(state): extract::State<Arc<super::State>>,- extract::Query(params): extract::Query<Params>,+ extract::Query(params): extract::Query<AliceParams>,) -> std::result::Result<Sse<impl Stream<Item = std::result::Result<axum::response::sse::Event, Infallible>>>,StatusCode,@@ -172,7 +182,7 @@ pub async fn route(.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?};- let params = entrypoint::alice::Params {+ let params = entrypoint::alice::AliceParams {message: params.message,prev_state: saved_state.map(|s| s.uuid),optic: params.optic,diff --git a/core/src/api/autosuggest.rs b/core/src/api/autosuggest.rsindex e6d4b2a4..d58649a6 100644--- a/core/src/api/autosuggest.rs+++ b/core/src/api/autosuggest.rs@@ -18,6 +18,7 @@ use std::{collections::HashMap, sync::Arc};use axum::{extract, response::IntoResponse, Json};use serde::Serialize;+use utoipa::{IntoParams, ToSchema};use super::State;@@ -38,13 +39,28 @@ fn highlight(query: &str, suggestion: &str) -> String {new_suggestion}-#[derive(Serialize)]-struct Suggestion {+#[derive(Serialize, ToSchema)]+#[serde(rename_all = "camelCase")]+pub struct Suggestion {highlighted: String,raw: String,}+#[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)]+#[serde(rename_all = "camelCase")]+pub struct AutosuggestQuery {+ q: String,+}+#[allow(clippy::unused_async)]+#[utoipa::path(+ post,+ path = "/beta/api/autosuggest",+ params(AutosuggestQuery),+ responses(+ (status = 200, description = "Autosuggest", body = Vec<Suggestion>),+ )+)]pub async fn route(extract::State(state): extract::State<Arc<State>>,extract::Query(params): extract::Query<HashMap<String, String>>,diff --git a/core/src/api/docs.rs b/core/src/api/docs.rsindex 0f890964..1e9755f3 100644--- a/core/src/api/docs.rs+++ b/core/src/api/docs.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use super::{search, webgraph};+use super::{alice, autosuggest, fact_check, search, summarize, webgraph};use axum::Router;use utoipa::{Modify, OpenApi};use utoipa_swagger_ui::SwaggerUi;@@ -24,7 +24,11 @@ use utoipa_swagger_ui::SwaggerUi;paths(search::api,webgraph::similar_sites,- webgraph::knows_site+ webgraph::knows_site,+ autosuggest::route,+ summarize::summarize_route,+ fact_check::fact_check_route,+ alice::alice_route,),components(schemas(@@ -55,6 +59,15 @@ use utoipa_swagger_ui::SwaggerUi;webgraph::SimilarSitesParams,webgraph::KnowsSite,crate::entrypoint::webgraph_server::ScoredSite,++ autosuggest::Suggestion,+ fact_check::FactCheckParams,+ fact_check::FactCheckResponse,++ crate::alice::SimplifiedWebsite,+ crate::alice::ExecutionState,+ crate::alice::EncodedEncryptedState,+ alice::EncodedSavedState,),),modifiers(&ApiModifier),diff --git a/core/src/api/fact_check.rs b/core/src/api/fact_check.rsindex 49359a11..3e5e686c 100644--- a/core/src/api/fact_check.rs+++ b/core/src/api/fact_check.rs@@ -17,26 +17,35 @@use std::sync::Arc;use axum::{extract, response::IntoResponse, Json};+use utoipa::ToSchema;-#[derive(serde::Deserialize, serde::Serialize, Debug)]-pub struct Params {+#[derive(serde::Deserialize, serde::Serialize, Debug, ToSchema)]+pub struct FactCheckParams {claim: String,evidence: String,}-#[derive(serde::Deserialize, serde::Serialize, Debug)]-pub struct Response {+#[derive(serde::Deserialize, serde::Serialize, Debug, ToSchema)]+pub struct FactCheckResponse {score: f64,}-pub async fn route(+#[utoipa::path(+ post,+ path = "/beta/api/fact_check",+ request_body(content = FactCheckParams),+ responses(+ (status = 200, description = "Fact check the given claim against the given evidence", body = FactCheckResponse),+ )+)]+pub async fn fact_check_route(extract::State(state): extract::State<Arc<super::State>>,- extract::Json(params): extract::Json<Params>,+ extract::Json(params): extract::Json<FactCheckParams>,) -> Result<impl IntoResponse, http::StatusCode> {let score = state.fact_checker.run(¶ms.claim, ¶ms.evidence).map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?;- Ok(Json(Response { score }))+ Ok(Json(FactCheckResponse { score }))}diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex a0e4dd15..0fde3970 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -201,24 +201,39 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route.route("/privacy-and-happy-lawyers", get(privacy::route)).route("/webmasters", get(crawler::info_route)).route("/opensearch.xml", get(opensearch::route))- .route("/improvement/click", post(improvement::click))- .route("/improvement/store", post(improvement::store))+ .merge(+ Router::new()+ .route("/improvement/click", post(improvement::click))+ .route("/improvement/store", post(improvement::store))+ .layer(cors_layer()),+ ).fallback(get_service(ServeDir::new("frontend/dist/"))).layer(CompressionLayer::new()).merge(docs::router()).nest("/beta",Router::new()- .route("/api/summarize", get(summarize::route))+ .route("/api/autosuggest", post(autosuggest::route))+ .route("/api/summarize", get(summarize::summarize_route)).route("/api/webgraph/similar_sites", post(webgraph::similar_sites))- .route("/api/webgraph/knows_site", get(webgraph::knows_site))- .route("/api/alice", get(alice::route))+ .route("/api/webgraph/knows_site", post(webgraph::knows_site))+ .route("/api/alice", get(alice::alice_route)).route("/api/alice/save_state", post(alice::save_state))- .route("/api/fact_check", post(fact_check::route)),+ .route("/api/fact_check", post(fact_check::fact_check_route))+ .layer(cors_layer()),).with_state(state))}+/// Enables CORS for development where the API and Deno frontend are on+/// different hosts.+fn cors_layer() -> tower_http::cors::CorsLayer {+ #[cfg(feature = "cors")]+ return tower_http::cors::CorsLayer::permissive();+ #[cfg(not(feature = "cors"))]+ tower_http::cors::CorsLayer::new()+}+pub fn metrics_router(registry: crate::metrics::PrometheusRegistry) -> Router {Router::new().route("/metrics", get(metrics::route))diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rsindex 083c5e18..b8656b3a 100644--- a/core/src/api/summarize.rs+++ b/core/src/api/summarize.rs@@ -25,12 +25,13 @@ use http::StatusCode;use serde::Deserialize;use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};use tokio_stream::StreamExt as _;+use utoipa::IntoParams;use super::State;use crate::Result;-#[derive(Deserialize, Debug)]-pub struct Params {+#[derive(Deserialize, Debug, IntoParams)]+pub struct SummarizeParams {pub url: String,pub query: String,}@@ -42,7 +43,7 @@ fn summarize_blocking(iter: impl Iterator<Item = String>, tx: UnboundedSender<St}async fn summarize(- params: Params,+ params: SummarizeParams,state: Arc<State>,) -> Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>> {let webpage = state.searcher.get_webpage(¶ms.url).await?;@@ -66,8 +67,16 @@ async fn summarize(}#[allow(clippy::unused_async)]-pub async fn route(- extract::Query(params): extract::Query<Params>,+#[utoipa::path(+ get,+ path = "/beta/api/summarize",+ params(SummarizeParams),+ responses(+ (status = 200, description = "Summarize a website", body = String),+ )+)]+pub async fn summarize_route(+ extract::Query(params): extract::Query<SummarizeParams>,extract::State(state): extract::State<Arc<State>>,) -> std::result::Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>, StatusCode>{diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rsindex bb3f6dfd..341cec34 100644--- a/core/src/entrypoint/alice.rs+++ b/core/src/entrypoint/alice.rs@@ -83,7 +83,7 @@ pub async fn save_state(#[derive(serde::Serialize, serde::Deserialize, Debug)]#[serde(rename_all = "camelCase")]-pub struct Params {+pub struct AliceParams {pub message: String,pub optic: Option<String>,pub prev_state: Option<uuid::Uuid>,@@ -91,7 +91,7 @@ pub struct Params {pub async fn route(extract::State(state): extract::State<Arc<State>>,- extract::Query(params): extract::Query<Params>,+ extract::Query(params): extract::Query<AliceParams>,) -> std::result::Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>,http::StatusCode,