web search done right
added webgraph endpoints to api
Mikkel Denker2b88af096e0c
diff --git a/configs/webgraph/server.toml b/configs/webgraph/server.tomlindex 2f537362..1ff98aa4 100644--- a/configs/webgraph/server.toml+++ b/configs/webgraph/server.toml@@ -1,5 +1,6 @@host = "0.0.0.0:3003"cluster_id = "dev_webgraph"gossip_addr = "0.0.0.0:3007"-graph_path = "data/webgraph_host"+host_graph_path = "data/webgraph_host"+page_graph_path = "data/webgraph_page"inbound_similarity_path = "data/centrality/inbound_similarity"diff --git a/core/src/api/docs.rs b/core/src/api/docs.rsindex 3242af08..e67fb588 100644--- a/core/src/api/docs.rs+++ b/core/src/api/docs.rs@@ -23,8 +23,12 @@ use utoipa_swagger_ui::SwaggerUi;#[openapi(paths(search::api,- webgraph::similar_sites,- webgraph::knows_site,+ webgraph::host::similar,+ webgraph::host::knows,+ webgraph::host::ingoing_hosts,+ webgraph::host::outgoing_hosts,+ webgraph::page::ingoing_pages,+ webgraph::page::outgoing_pages,autosuggest::route,summarize::summarize_route,fact_check::fact_check_route,@@ -64,7 +68,7 @@ use utoipa_swagger_ui::SwaggerUi;crate::bangs::BangHit,crate::bangs::Bang,- webgraph::SimilarSitesParams,+ webgraph::host::SimilarSitesParams,webgraph::KnowsSite,crate::entrypoint::webgraph_server::ScoredSite,@@ -78,6 +82,9 @@ use utoipa_swagger_ui::SwaggerUi;alice::EncodedSavedState,sites::SitesExportOpticParams,explore::ExploreExportOpticParams,++ crate::webgraph::Node,+ crate::webgraph::FullEdge,),),modifiers(&ApiModifier),diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 0ffa1e5f..6bf2f241 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -171,8 +171,24 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {.route("/api/autosuggest", post(autosuggest::route)).route("/api/autosuggest/browser", get(autosuggest::browser)).route("/api/summarize", get(summarize::summarize_route))- .route("/api/webgraph/similar_sites", post(webgraph::similar_sites))- .route("/api/webgraph/knows_site", post(webgraph::knows_site))+ .route("/api/webgraph/host/similar", post(webgraph::host::similar))+ .route("/api/webgraph/host/knows", post(webgraph::host::knows))+ .route(+ "/api/webgraph/host/ingoing",+ post(webgraph::host::ingoing_hosts),+ )+ .route(+ "/api/webgraph/host/outgoing",+ post(webgraph::host::outgoing_hosts),+ )+ .route(+ "/api/webgraph/page/ingoing",+ post(webgraph::page::ingoing_pages),+ )+ .route(+ "/api/webgraph/page/outgoing",+ post(webgraph::page::outgoing_pages),+ ).route("/api/alice", get(alice::alice_route)).route("/api/alice/save_state", post(alice::save_state)).route("/api/fact_check", post(fact_check::fact_check_route))diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rsindex 3dceac08..bc57c76c 100644--- a/core/src/api/webgraph.rs+++ b/core/src/api/webgraph.rs@@ -20,8 +20,10 @@ use axum::{extract, response::IntoResponse, Json};use http::StatusCode;use utoipa::{IntoParams, ToSchema};-use crate::distributed::{- cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff, sonic,+use crate::{+ distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff, sonic},+ entrypoint::webgraph_server::GraphLevel,+ webgraph::{FullEdge, Node},};use super::State;@@ -47,31 +49,231 @@ impl RemoteWebgraph {}}-#[derive(serde::Deserialize, ToSchema)]-#[serde(rename_all = "camelCase")]-pub struct SimilarSitesParams {- pub sites: Vec<String>,- pub top_n: usize,+pub mod host {+ use super::*;++ #[derive(serde::Deserialize, ToSchema)]+ #[serde(rename_all = "camelCase")]+ pub struct SimilarSitesParams {+ pub sites: Vec<String>,+ pub top_n: usize,+ }++ #[derive(serde::Deserialize, IntoParams)]+ #[serde(rename_all = "camelCase")]+ pub struct KnowsSiteParams {+ pub site: String,+ }++ #[derive(serde::Deserialize, IntoParams)]+ #[serde(rename_all = "camelCase")]+ pub struct HostLinksParams {+ pub site: String,+ }++ #[allow(clippy::unused_async)]+ #[utoipa::path(post,+ path = "/beta/api/webgraph/host/similar",+ request_body(content = SimilarSitesParams),+ responses(+ (status = 200, description = "List of similar sites", body = Vec<ScoredSite>),+ )+ )]+ pub async fn similar(+ extract::State(state): extract::State<Arc<State>>,+ extract::Json(params): extract::Json<SimilarSitesParams>,+ ) -> std::result::Result<impl IntoResponse, StatusCode> {+ state.counters.explore_counter.inc();+ let host = state+ .remote_webgraph+ .host()+ .await+ .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;++ let retry = ExponentialBackoff::from_millis(30)+ .with_limit(Duration::from_millis(200))+ .take(5);++ let conn = sonic::service::ResilientConnection::create_with_timeout(+ host,+ Duration::from_secs(30),+ retry,+ )+ .await+ .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;++ match conn+ .send_with_timeout(+ &crate::entrypoint::webgraph_server::SimilarSites {+ sites: params.sites,+ top_n: params.top_n,+ },+ Duration::from_secs(60),+ )+ .await+ {+ Ok(nodes) => Ok(Json(nodes)),+ Err(err) => {+ tracing::error!("Failed to send request to webgraph: {}", err);+ Err(StatusCode::INTERNAL_SERVER_ERROR)+ }+ }+ }++ #[utoipa::path(post,+ path = "/beta/api/webgraph/host/knows",+ params(KnowsSiteParams),+ responses(+ (status = 200, description = "Whether the site is known", body = KnowsSite),+ )+ )]+ pub async fn knows(+ extract::State(state): extract::State<Arc<State>>,+ extract::Query(params): extract::Query<KnowsSiteParams>,+ ) -> std::result::Result<impl IntoResponse, StatusCode> {+ let host = state+ .remote_webgraph+ .host()+ .await+ .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;++ let retry = ExponentialBackoff::from_millis(30)+ .with_limit(Duration::from_millis(200))+ .take(5);++ let conn = sonic::service::ResilientConnection::create_with_timeout(+ host,+ Duration::from_secs(30),+ retry,+ )+ .await+ .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;++ match conn+ .send_with_timeout(+ &crate::entrypoint::webgraph_server::Knows { site: params.site },+ Duration::from_secs(60),+ )+ .await+ {+ Ok(Some(node)) => Ok(Json(KnowsSite::Known { site: node.name })),+ Err(err) => {+ tracing::error!("Failed to send request to webgraph: {}", err);+ Ok(Json(KnowsSite::Unknown))+ }+ _ => Ok(Json(KnowsSite::Unknown)),+ }+ }++ #[utoipa::path(post,+ path = "/beta/api/webgraph/host/ingoing",+ params(HostLinksParams),+ responses(+ (status = 200, description = "Incoming links for a particular host", body = Vec<FullEdge>),+ )+ )]+ pub async fn ingoing_hosts(+ extract::State(state): extract::State<Arc<State>>,+ extract::Query(params): extract::Query<HostLinksParams>,+ ) -> std::result::Result<impl IntoResponse, StatusCode> {+ let node = Node::from(params.site).into_host();+ let links = ingoing_links(state, node, GraphLevel::Host)+ .await+ .map_err(|_| {+ tracing::error!("Failed to send request to webgraph");+ StatusCode::INTERNAL_SERVER_ERROR+ })?;++ Ok(Json(links))+ }++ #[utoipa::path(post,+ path = "/beta/api/webgraph/host/outgoing",+ params(HostLinksParams),+ responses(+ (status = 200, description = "Outgoing links for a particular host", body = Vec<FullEdge>),+ )+ )]+ pub async fn outgoing_hosts(+ extract::State(state): extract::State<Arc<State>>,+ extract::Query(params): extract::Query<HostLinksParams>,+ ) -> std::result::Result<impl IntoResponse, StatusCode> {+ let node = Node::from(params.site).into_host();+ let links = outgoing_links(state, node, GraphLevel::Host)+ .await+ .map_err(|_| {+ tracing::error!("Failed to send request to webgraph");+ StatusCode::INTERNAL_SERVER_ERROR+ })?;++ Ok(Json(links))+ }}-#[allow(clippy::unused_async)]-#[utoipa::path(post,- path = "/beta/api/webgraph/similar_sites",- request_body(content = SimilarSitesParams),- responses(- (status = 200, description = "List of similar sites", body = Vec<ScoredSite>),- )-)]-pub async fn similar_sites(- extract::State(state): extract::State<Arc<State>>,- extract::Json(params): extract::Json<SimilarSitesParams>,-) -> std::result::Result<impl IntoResponse, StatusCode> {- state.counters.explore_counter.inc();+pub mod page {+ use super::*;++ #[derive(serde::Deserialize, IntoParams)]+ #[serde(rename_all = "camelCase")]+ pub struct PageLinksParams {+ pub page: String,+ }++ #[utoipa::path(post,+ path = "/beta/api/webgraph/page/ingoing",+ params(PageLinksParams),+ responses(+ (status = 200, description = "Incoming links for a particular page", body = Vec<FullEdge>),+ )+ )]+ pub async fn ingoing_pages(+ extract::State(state): extract::State<Arc<State>>,+ extract::Query(params): extract::Query<PageLinksParams>,+ ) -> std::result::Result<impl IntoResponse, StatusCode> {+ let node = Node::from(params.page);+ let links = ingoing_links(state, node, GraphLevel::Page)+ .await+ .map_err(|_| {+ tracing::error!("Failed to send request to webgraph");+ StatusCode::INTERNAL_SERVER_ERROR+ })?;++ Ok(Json(links))+ }++ #[utoipa::path(post,+ path = "/beta/api/webgraph/page/outgoing",+ params(PageLinksParams),+ responses(+ (status = 200, description = "Outgoing links for a particular page", body = Vec<FullEdge>),+ )+ )]+ pub async fn outgoing_pages(+ extract::State(state): extract::State<Arc<State>>,+ extract::Query(params): extract::Query<PageLinksParams>,+ ) -> std::result::Result<impl IntoResponse, StatusCode> {+ let node = Node::from(params.page);+ let links = outgoing_links(state, node, GraphLevel::Page)+ .await+ .map_err(|_| {+ tracing::error!("Failed to send request to webgraph");+ StatusCode::INTERNAL_SERVER_ERROR+ })?;++ Ok(Json(links))+ }+}++async fn ingoing_links(+ state: Arc<State>,+ node: Node,+ level: GraphLevel,+) -> anyhow::Result<Vec<FullEdge>> {let host = state.remote_webgraph.host().await- .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;+ .ok_or(anyhow::anyhow!("no remote webgraph"))?;let retry = ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200))@@ -82,56 +284,26 @@ pub async fn similar_sites(Duration::from_secs(30),retry,)- .await- .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;+ .await?;- match conn+ Ok(conn.send_with_timeout(- &crate::entrypoint::webgraph_server::SimilarSites {- sites: params.sites,- top_n: params.top_n,- },+ &crate::entrypoint::webgraph_server::IngoingLinks { node, level },Duration::from_secs(60),)- .await- {- Ok(nodes) => Ok(Json(nodes)),- Err(err) => {- tracing::error!("Failed to send request to webgraph: {}", err);- Err(StatusCode::INTERNAL_SERVER_ERROR)- }- }-}--#[derive(serde::Deserialize, IntoParams)]-pub struct KnowsSiteParams {- pub site: String,-}--#[derive(serde::Serialize, serde::Deserialize, ToSchema)]-#[serde(tag = "type", rename_all = "camelCase")]-pub enum KnowsSite {- Known { site: String },- Unknown,+ .await?)}-#[allow(clippy::unused_async)]-#[utoipa::path(post,- path = "/beta/api/webgraph/knows_site",- params(KnowsSiteParams),- responses(- (status = 200, description = "Whether the site is known", body = KnowsSite),- )-)]-pub async fn knows_site(- extract::State(state): extract::State<Arc<State>>,- extract::Query(params): extract::Query<KnowsSiteParams>,-) -> std::result::Result<impl IntoResponse, StatusCode> {+async fn outgoing_links(+ state: Arc<State>,+ node: Node,+ level: GraphLevel,+) -> anyhow::Result<Vec<FullEdge>> {let host = state.remote_webgraph.host().await- .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;+ .ok_or(anyhow::anyhow!("no remote webgraph"))?;let retry = ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200))@@ -142,21 +314,19 @@ pub async fn knows_site(Duration::from_secs(30),retry,)- .await- .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;+ .await?;- match conn+ Ok(conn.send_with_timeout(- &crate::entrypoint::webgraph_server::Knows { site: params.site },+ &crate::entrypoint::webgraph_server::OutgoingLinks { node, level },Duration::from_secs(60),)- .await- {- Ok(Some(node)) => Ok(Json(KnowsSite::Known { site: node.name })),- Err(err) => {- tracing::error!("Failed to send request to webgraph: {}", err);- Ok(Json(KnowsSite::Unknown))- }- _ => Ok(Json(KnowsSite::Unknown)),- }+ .await?)+}++#[derive(serde::Serialize, serde::Deserialize, ToSchema)]+#[serde(tag = "type", rename_all = "camelCase")]+pub enum KnowsSite {+ Known { site: String },+ Unknown,}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex b67f111c..caaa7be2 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -344,7 +344,8 @@ pub struct AliceLocalConfig {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct WebgraphServerConfig {pub host: SocketAddr,- pub graph_path: String,+ pub host_graph_path: String,+ pub page_graph_path: String,pub inbound_similarity_path: String,pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex 97cff0a7..3c075e0d 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -34,7 +34,9 @@ use crate::ranking::inbound_similarity::InboundSimilarity;use crate::searcher::DistributedSearcher;use crate::similar_sites::SimilarSitesFinder;use crate::sonic_service;+use crate::webgraph::FullEdge;use crate::webgraph::Node;+use crate::webgraph::Webgraph;use crate::webgraph::WebgraphBuilder;use crate::Result;@@ -51,15 +53,21 @@ const MAX_SITES: usize = 20;pub struct WebGraphService {searcher: DistributedSearcher,similar_sites_finder: SimilarSitesFinder,+ host_graph: Arc<Webgraph>,+ page_graph: Arc<Webgraph>,}-sonic_service!(WebGraphService, [SimilarSites, Knows]);+sonic_service!(+ WebGraphService,+ [SimilarSites, Knows, IngoingLinks, OutgoingLinks]+);#[derive(Debug, Clone, Serialize, Deserialize)]pub struct SimilarSites {pub sites: Vec<String>,pub top_n: usize,}+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct Knows {pub site: String,@@ -116,6 +124,54 @@ impl Message<WebGraphService> for Knows {}}+#[derive(Debug, Clone, Serialize, Deserialize)]+pub enum GraphLevel {+ Host,+ Page,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct IngoingLinks {+ pub node: Node,+ pub level: GraphLevel,+}++#[async_trait::async_trait]+impl Message<WebGraphService> for IngoingLinks {+ type Response = Vec<FullEdge>;++ async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {+ match self.level {+ GraphLevel::Host => {+ let node = self.node.into_host();+ Ok(server.host_graph.ingoing_edges(node))+ }+ GraphLevel::Page => Ok(server.page_graph.ingoing_edges(self.node)),+ }+ }+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct OutgoingLinks {+ pub node: Node,+ pub level: GraphLevel,+}++#[async_trait::async_trait]+impl Message<WebGraphService> for OutgoingLinks {+ type Response = Vec<FullEdge>;++ async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {+ match self.level {+ GraphLevel::Host => {+ let node = self.node.into_host();+ Ok(server.host_graph.outgoing_edges(node))+ }+ GraphLevel::Page => Ok(server.page_graph.outgoing_edges(self.node)),+ }+ }+}+pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {let addr: SocketAddr = config.host;@@ -133,16 +189,19 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {);let searcher = DistributedSearcher::new(cluster);- let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());+ let host_graph = Arc::new(WebgraphBuilder::new(config.host_graph_path).open());+ let page_graph = Arc::new(WebgraphBuilder::new(config.page_graph_path).open());let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;let similar_sites_finder = SimilarSitesFinder::new(- Arc::clone(&graph),+ Arc::clone(&host_graph),inbound_similarity,config.max_similar_sites,);let server = WebGraphService {+ host_graph,+ page_graph,searcher,similar_sites_finder,}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 81fcb225..7fa72b00 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -26,6 +26,7 @@ use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};use url::Url;+use utoipa::ToSchema;use crate::executor::Executor;use crate::intmap;@@ -114,14 +115,16 @@ wherepub label: L,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct FullEdge {pub from: Node,pub to: Node,pub label: String,}-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct Node {pub name: String,}diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex 3439d834..a22f7ca8 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -17,11 +17,11 @@ export const api = {},similarSites: (- props: Props<'/beta/api/webgraph/similar_sites', 'post'>,+ props: Props<'/beta/api/webgraph/host/similar', 'post'>,options?: ApiOptions,) => {const { data, cancel } = send(- '/beta/api/webgraph/similar_sites',+ '/beta/api/webgraph/host/similar','post','json',props,@@ -30,9 +30,9 @@ export const api = {return { data: data.then((res) => res[200]), cancel };},- knowsSite: (props: Props<'/beta/api/webgraph/knows_site', 'post'>, options?: ApiOptions) => {+ knowsSite: (props: Props<'/beta/api/webgraph/host/knows', 'post'>, options?: ApiOptions) => {const { data, cancel } = send(- '/beta/api/webgraph/knows_site',+ '/beta/api/webgraph/host/knows','post','parameters',props,diff --git a/frontend/src/lib/api/schema.d.ts b/frontend/src/lib/api/schema.d.tsindex 96627d13..798f3545 100644--- a/frontend/src/lib/api/schema.d.ts+++ b/frontend/src/lib/api/schema.d.ts@@ -31,11 +31,23 @@ export interface paths {"/beta/api/summarize": {get: operations["summarize_route"];};- "/beta/api/webgraph/knows_site": {- post: operations["knows_site"];+ "/beta/api/webgraph/host/ingoing": {+ post: operations["ingoing_hosts"];};- "/beta/api/webgraph/similar_sites": {- post: operations["similar_sites"];+ "/beta/api/webgraph/host/knows": {+ post: operations["knows"];+ };+ "/beta/api/webgraph/host/outgoing": {+ post: operations["outgoing_hosts"];+ };+ "/beta/api/webgraph/host/similar": {+ post: operations["similar"];+ };+ "/beta/api/webgraph/page/ingoing": {+ post: operations["ingoing_pages"];+ };+ "/beta/api/webgraph/page/outgoing": {+ post: operations["outgoing_pages"];};}@@ -187,6 +199,11 @@ export interface components {/** Format: double */score: number;};+ FullEdge: {+ from: components["schemas"]["Node"];+ label: string;+ to: components["schemas"]["Node"];+ };HighlightedSpellCorrection: {highlighted: string;raw: string;@@ -199,6 +216,9 @@ export interface components {/** @enum {string} */type: "unknown";};+ Node: {+ name: string;+ };/** @enum {string} */Region: "All" | "Denmark" | "France" | "Germany" | "Spain" | "US";ScoredSite: {@@ -407,7 +427,22 @@ export interface operations {};};};- knows_site: {+ ingoing_hosts: {+ parameters: {+ query: {+ site: string;+ };+ };+ responses: {+ /** @description Incoming links for a particular host */+ 200: {+ content: {+ "application/json": components["schemas"]["FullEdge"][];+ };+ };+ };+ };+ knows: {parameters: {query: {site: string;@@ -422,7 +457,22 @@ export interface operations {};};};- similar_sites: {+ outgoing_hosts: {+ parameters: {+ query: {+ site: string;+ };+ };+ responses: {+ /** @description Outgoing links for a particular host */+ 200: {+ content: {+ "application/json": components["schemas"]["FullEdge"][];+ };+ };+ };+ };+ similar: {requestBody: {content: {"application/json": components["schemas"]["SimilarSitesParams"];@@ -437,4 +487,34 @@ export interface operations {};};};+ ingoing_pages: {+ parameters: {+ query: {+ page: string;+ };+ };+ responses: {+ /** @description Incoming links for a particular page */+ 200: {+ content: {+ "application/json": components["schemas"]["FullEdge"][];+ };+ };+ };+ };+ outgoing_pages: {+ parameters: {+ query: {+ page: string;+ };+ };+ responses: {+ /** @description Outgoing links for a particular page */+ 200: {+ content: {+ "application/json": components["schemas"]["FullEdge"][];+ };+ };+ };+ };}