Skip to content

web search done right

6.6 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use std::net::SocketAddr;
use std::sync::Arc;

use tracing::info;
use utoipa::ToSchema;

use crate::config;
use crate::distributed::cluster::Cluster;
use crate::distributed::member::Member;
use crate::distributed::member::Service;
use crate::distributed::sonic;
use crate::distributed::sonic::service::sonic_service;
use crate::distributed::sonic::service::Message;
use crate::webgraph;
use crate::webgraph::query::BacklinksQuery;
use crate::webgraph::query::BacklinksWithLabelsQuery;
use crate::webgraph::query::ForwardlinksQuery;
use crate::webgraph::query::FullBacklinksQuery;
use crate::webgraph::query::FullForwardlinksQuery;
use crate::webgraph::query::FullHostBacklinksQuery;
use crate::webgraph::query::FullHostForwardlinksQuery;
use crate::webgraph::query::FullLinksBetweenQuery;
use crate::webgraph::query::HostBacklinksQuery;
use crate::webgraph::query::HostForwardlinksQuery;
use crate::webgraph::query::HostGroupQuery;
use crate::webgraph::query::HostGroupSketchQuery;
use crate::webgraph::query::Id2NodeQuery;
use crate::webgraph::NodeID;
use crate::webgraph::Webgraph;
use crate::webgraph::WebgraphBuilder;
use crate::Result;

#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct ScoredHost {
    pub host: String,
    pub score: f64,
    pub description: Option<String>,
}

pub struct WebGraphService {
    graph: Arc<Webgraph>,
}

pub trait RetrieveReq: bincode::Encode + bincode::Decode + Clone {
    type Query: webgraph::Query + bincode::Encode + bincode::Decode;
    fn new(
        query: Self::Query,
        fruit: <<Self::Query as webgraph::Query>::Collector as webgraph::Collector>::Fruit,
    ) -> Self;
}

pub trait Query
where
    Self: webgraph::Query
        + bincode::Encode
        + bincode::Decode
        + sonic::service::Wrapper<WebGraphService>,
{
    type RetrieveReq: RetrieveReq<Query = Self>;
}

#[derive(bincode::Encode, bincode::Decode, Clone)]
pub struct EncodedError {
    pub msg: String,
}

impl std::fmt::Display for EncodedError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.msg)
    }
}

macro_rules! impl_search {
    ([$($q:ident),*$(,)?]) => {
        $(
            impl Message<WebGraphService> for $q {
                type Response = Result<<<$q as webgraph::Query>::Collector as webgraph::Collector>::Fruit, EncodedError>;

                async fn handle(self, server: &WebGraphService) -> Self::Response {
                    let graph = Arc::clone(&server.graph);
                    tokio::task::spawn_blocking(move || {
                        graph.search_initial(&self).map_err(|e| EncodedError { msg: e.to_string() })
                    })
                    .await
                    .unwrap()
                }
            }

            paste::item! {
                #[derive(bincode::Encode, bincode::Decode, Clone)]
                pub struct [<$q Retrieve>] {
                    pub query: $q,
                    #[bincode(with_serde)]
                    pub fruit: <<$q as webgraph::Query>::Collector as webgraph::Collector>::Fruit,
                }

                impl Message<WebGraphService> for [<$q Retrieve>] {
                    type Response = Result<<$q as webgraph::Query>::IntermediateOutput, EncodedError>;
                    async fn handle(self, server: &WebGraphService) -> Self::Response {
                        let graph = Arc::clone(&server.graph);
                        tokio::task::spawn_blocking(move || {
                            graph.retrieve(&self.query, self.fruit).map_err(|e| EncodedError { msg: e.to_string() })
                        })
                        .await
                        .unwrap()
                    }
                }

                impl Query for $q {
                    type RetrieveReq = [<$q Retrieve>];
                }

                impl RetrieveReq for [<$q Retrieve>] {
                    type Query = $q;
                    fn new(query: Self::Query, fruit: <<Self::Query as webgraph::Query>::Collector as webgraph::Collector>::Fruit) -> Self {
                        Self { query, fruit }
                    }
                }
            }
        )*

        paste::item! {
            sonic_service!(WebGraphService, [GetPageNodeIDs, $(
                $q,
                [<$q Retrieve>],
            )*]);
        }

    }
}

impl_search!([
    BacklinksQuery,
    HostBacklinksQuery,
    FullBacklinksQuery,
    FullHostBacklinksQuery,
    BacklinksWithLabelsQuery,
    Id2NodeQuery,
    ForwardlinksQuery,
    HostForwardlinksQuery,
    FullForwardlinksQuery,
    FullHostForwardlinksQuery,
    HostGroupSketchQuery,
    HostGroupQuery,
    FullLinksBetweenQuery,
]);

#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct GetPageNodeIDs {
    pub offset: u64,
    pub limit: u64,
}

impl Message<WebGraphService> for GetPageNodeIDs {
    type Response = Vec<NodeID>;

    async fn handle(self, server: &WebGraphService) -> Self::Response {
        server
            .graph
            .page_node_ids_with_offset(self.offset, self.limit)
    }
}

pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
    let addr: SocketAddr = config.host;

    let graph = Arc::new(WebgraphBuilder::new(config.graph_path, config.shard).open()?);

    let server = WebGraphService { graph }.bind(addr).await.unwrap();

    // dropping the handle leaves the cluster
    let _cluster = Arc::new(
        Cluster::join(
            Member::new(Service::Webgraph {
                host: addr,
                shard: config.shard,
            }),
            config.gossip_addr,
            config.gossip_seed_nodes.unwrap_or_default(),
        )
        .await?,
    );

    info!("webgraph server is ready to accept requests on {}", addr);

    loop {
        if let Err(e) = server.accept().await {
            tracing::error!("{:?}", e);
        }
    }
}