Skip to content

web search done right

Add more endpoints to the OpenAPI specification (#80)

Oliver Bøvingf0f0c420bbc8
diff --git a/Cargo.toml b/Cargo.toml
index d7bea3a8..93b0dedb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,7 +56,7 @@ memmap = "0.7.0"
once_cell = "1.13.1"
regex = "1.6.0"
http = "0.2.8"
-tower-http = { version = "0.4.0", features = ["compression-gzip"] }
+tower-http = { version = "0.4.0", features = ["compression-gzip", "cors"] }
lalrpop-util = { version = "0.19.8", features = ["lexer"] }
lalrpop = { version = "0.19.8", features = ["lexer"] }
serde_urlencoded = "0.7.1"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 680c9a85..9faccb4f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -9,7 +9,9 @@ default-run = "stract"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
-dev = []
+default = ["cors"]
+dev = ["cors"]
+cors = []
[[bin]]
name = "stract"
@@ -147,4 +149,4 @@ harness = false
[[bench]]
name = "naive-bayes"
-harness = false
\ No newline at end of file
+harness = false
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index 568c7cd4..73e13214 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -40,6 +40,7 @@ use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};
use half::bf16;
use tch::Tensor;
use url::Url;
+use utoipa::ToSchema;
use crate::{
api::search::ApiSearchQuery,
@@ -90,7 +91,7 @@ pub enum Error {
LastMessageNotUser,
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, ToSchema)]
pub struct SimplifiedWebsite {
pub title: String,
pub text: String,
@@ -112,7 +113,7 @@ impl SimplifiedWebsite {
}
}
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
#[serde(tag = "type", rename_all = "camelCase")]
pub enum ExecutionState {
BeginSearch {
@@ -402,7 +403,7 @@ impl Tokenizer {
}
}
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
/// base64 encoded `EncryptedState`
pub struct EncodedEncryptedState(String);
diff --git a/core/src/api/alice.rs b/core/src/api/alice.rs
index 7d8dbe62..59579e38 100644
--- a/core/src/api/alice.rs
+++ b/core/src/api/alice.rs
@@ -24,6 +24,7 @@ use eventsource_stream::Eventsource;
use http::StatusCode;
use rand::seq::SliceRandom;
use tokio_stream::{Stream, StreamExt};
+use utoipa::{IntoParams, ToSchema};
use crate::{
distributed::member::Service,
@@ -91,7 +92,7 @@ pub async fn save_state(
Ok(EncodedSavedState::encode(saved_state).0)
}
-#[derive(serde::Serialize, serde::Deserialize, Debug)]
+#[derive(serde::Serialize, serde::Deserialize, Debug, ToSchema)]
pub struct EncodedSavedState(String);
impl EncodedSavedState {
@@ -108,17 +109,26 @@ impl EncodedSavedState {
}
}
-#[derive(serde::Serialize, serde::Deserialize, Debug)]
+#[derive(serde::Serialize, serde::Deserialize, Debug, IntoParams)]
#[serde(rename_all = "camelCase")]
-pub struct Params {
+pub struct AliceParams {
message: String,
optic: Option<String>,
prev_state: Option<EncodedSavedState>,
}
-pub async fn route(
+#[allow(clippy::unused_async)]
+#[utoipa::path(
+ get,
+ path = "/beta/api/alice",
+ params(AliceParams),
+ responses(
+ (status = 200, description = "Interact with Alice", body = ExecutionState),
+ )
+)]
+pub async fn alice_route(
extract::State(state): extract::State<Arc<super::State>>,
- extract::Query(params): extract::Query<Params>,
+ extract::Query(params): extract::Query<AliceParams>,
) -> std::result::Result<
Sse<impl Stream<Item = std::result::Result<axum::response::sse::Event, Infallible>>>,
StatusCode,
@@ -172,7 +182,7 @@ pub async fn route(
.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?
};
- let params = entrypoint::alice::Params {
+ let params = entrypoint::alice::AliceParams {
message: params.message,
prev_state: saved_state.map(|s| s.uuid),
optic: params.optic,
diff --git a/core/src/api/autosuggest.rs b/core/src/api/autosuggest.rs
index e6d4b2a4..d58649a6 100644
--- a/core/src/api/autosuggest.rs
+++ b/core/src/api/autosuggest.rs
@@ -18,6 +18,7 @@ use std::{collections::HashMap, sync::Arc};
use axum::{extract, response::IntoResponse, Json};
use serde::Serialize;
+use utoipa::{IntoParams, ToSchema};
use super::State;
@@ -38,13 +39,28 @@ fn highlight(query: &str, suggestion: &str) -> String {
new_suggestion
}
-#[derive(Serialize)]
-struct Suggestion {
+#[derive(Serialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Suggestion {
highlighted: String,
raw: String,
}
+#[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)]
+#[serde(rename_all = "camelCase")]
+pub struct AutosuggestQuery {
+ q: String,
+}
+
#[allow(clippy::unused_async)]
+#[utoipa::path(
+ post,
+ path = "/beta/api/autosuggest",
+ params(AutosuggestQuery),
+ responses(
+ (status = 200, description = "Autosuggest", body = Vec<Suggestion>),
+ )
+)]
pub async fn route(
extract::State(state): extract::State<Arc<State>>,
extract::Query(params): extract::Query<HashMap<String, String>>,
diff --git a/core/src/api/docs.rs b/core/src/api/docs.rs
index 0f890964..1e9755f3 100644
--- a/core/src/api/docs.rs
+++ b/core/src/api/docs.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use super::{search, webgraph};
+use super::{alice, autosuggest, fact_check, search, summarize, webgraph};
use axum::Router;
use utoipa::{Modify, OpenApi};
use utoipa_swagger_ui::SwaggerUi;
@@ -24,7 +24,11 @@ use utoipa_swagger_ui::SwaggerUi;
paths(
search::api,
webgraph::similar_sites,
- webgraph::knows_site
+ webgraph::knows_site,
+ autosuggest::route,
+ summarize::summarize_route,
+ fact_check::fact_check_route,
+ alice::alice_route,
),
components(
schemas(
@@ -55,6 +59,15 @@ use utoipa_swagger_ui::SwaggerUi;
webgraph::SimilarSitesParams,
webgraph::KnowsSite,
crate::entrypoint::webgraph_server::ScoredSite,
+
+ autosuggest::Suggestion,
+ fact_check::FactCheckParams,
+ fact_check::FactCheckResponse,
+
+ crate::alice::SimplifiedWebsite,
+ crate::alice::ExecutionState,
+ crate::alice::EncodedEncryptedState,
+ alice::EncodedSavedState,
),
),
modifiers(&ApiModifier),
diff --git a/core/src/api/fact_check.rs b/core/src/api/fact_check.rs
index 49359a11..3e5e686c 100644
--- a/core/src/api/fact_check.rs
+++ b/core/src/api/fact_check.rs
@@ -17,26 +17,35 @@
use std::sync::Arc;
use axum::{extract, response::IntoResponse, Json};
+use utoipa::ToSchema;
-#[derive(serde::Deserialize, serde::Serialize, Debug)]
-pub struct Params {
+#[derive(serde::Deserialize, serde::Serialize, Debug, ToSchema)]
+pub struct FactCheckParams {
claim: String,
evidence: String,
}
-#[derive(serde::Deserialize, serde::Serialize, Debug)]
-pub struct Response {
+#[derive(serde::Deserialize, serde::Serialize, Debug, ToSchema)]
+pub struct FactCheckResponse {
score: f64,
}
-pub async fn route(
+#[utoipa::path(
+ post,
+ path = "/beta/api/fact_check",
+ request_body(content = FactCheckParams),
+ responses(
+ (status = 200, description = "Fact check the given claim against the given evidence", body = FactCheckResponse),
+ )
+)]
+pub async fn fact_check_route(
extract::State(state): extract::State<Arc<super::State>>,
- extract::Json(params): extract::Json<Params>,
+ extract::Json(params): extract::Json<FactCheckParams>,
) -> Result<impl IntoResponse, http::StatusCode> {
let score = state
.fact_checker
.run(&params.claim, &params.evidence)
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?;
- Ok(Json(Response { score }))
+ Ok(Json(FactCheckResponse { score }))
}
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index a0e4dd15..0fde3970 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -201,24 +201,39 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route
.route("/privacy-and-happy-lawyers", get(privacy::route))
.route("/webmasters", get(crawler::info_route))
.route("/opensearch.xml", get(opensearch::route))
- .route("/improvement/click", post(improvement::click))
- .route("/improvement/store", post(improvement::store))
+ .merge(
+ Router::new()
+ .route("/improvement/click", post(improvement::click))
+ .route("/improvement/store", post(improvement::store))
+ .layer(cors_layer()),
+ )
.fallback(get_service(ServeDir::new("frontend/dist/")))
.layer(CompressionLayer::new())
.merge(docs::router())
.nest(
"/beta",
Router::new()
- .route("/api/summarize", get(summarize::route))
+ .route("/api/autosuggest", post(autosuggest::route))
+ .route("/api/summarize", get(summarize::summarize_route))
.route("/api/webgraph/similar_sites", post(webgraph::similar_sites))
- .route("/api/webgraph/knows_site", get(webgraph::knows_site))
- .route("/api/alice", get(alice::route))
+ .route("/api/webgraph/knows_site", post(webgraph::knows_site))
+ .route("/api/alice", get(alice::alice_route))
.route("/api/alice/save_state", post(alice::save_state))
- .route("/api/fact_check", post(fact_check::route)),
+ .route("/api/fact_check", post(fact_check::fact_check_route))
+ .layer(cors_layer()),
)
.with_state(state))
}
+/// Enables CORS for development where the API and Deno frontend are on
+/// different hosts.
+fn cors_layer() -> tower_http::cors::CorsLayer {
+ #[cfg(feature = "cors")]
+ return tower_http::cors::CorsLayer::permissive();
+ #[cfg(not(feature = "cors"))]
+ tower_http::cors::CorsLayer::new()
+}
+
pub fn metrics_router(registry: crate::metrics::PrometheusRegistry) -> Router {
Router::new()
.route("/metrics", get(metrics::route))
diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rs
index 083c5e18..b8656b3a 100644
--- a/core/src/api/summarize.rs
+++ b/core/src/api/summarize.rs
@@ -25,12 +25,13 @@ use http::StatusCode;
use serde::Deserialize;
use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
use tokio_stream::StreamExt as _;
+use utoipa::IntoParams;
use super::State;
use crate::Result;
-#[derive(Deserialize, Debug)]
-pub struct Params {
+#[derive(Deserialize, Debug, IntoParams)]
+pub struct SummarizeParams {
pub url: String,
pub query: String,
}
@@ -42,7 +43,7 @@ fn summarize_blocking(iter: impl Iterator<Item = String>, tx: UnboundedSender<St
}
async fn summarize(
- params: Params,
+ params: SummarizeParams,
state: Arc<State>,
) -> Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>> {
let webpage = state.searcher.get_webpage(&params.url).await?;
@@ -66,8 +67,16 @@ async fn summarize(
}
#[allow(clippy::unused_async)]
-pub async fn route(
- extract::Query(params): extract::Query<Params>,
+#[utoipa::path(
+ get,
+ path = "/beta/api/summarize",
+ params(SummarizeParams),
+ responses(
+ (status = 200, description = "Summarize a website", body = String),
+ )
+)]
+pub async fn summarize_route(
+ extract::Query(params): extract::Query<SummarizeParams>,
extract::State(state): extract::State<Arc<State>>,
) -> std::result::Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>, StatusCode>
{
diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rs
index bb3f6dfd..341cec34 100644
--- a/core/src/entrypoint/alice.rs
+++ b/core/src/entrypoint/alice.rs
@@ -83,7 +83,7 @@ pub async fn save_state(
#[derive(serde::Serialize, serde::Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
-pub struct Params {
+pub struct AliceParams {
pub message: String,
pub optic: Option<String>,
pub prev_state: Option<uuid::Uuid>,
@@ -91,7 +91,7 @@ pub struct Params {
pub async fn route(
extract::State(state): extract::State<Arc<State>>,
- extract::Query(params): extract::Query<Params>,
+ extract::Query(params): extract::Query<AliceParams>,
) -> std::result::Result<
Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>,
http::StatusCode,