web search done right
some module level docs
Mikkel Denkercfe9d0ca79c1
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex df58767c..f9fb7c15 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -14,6 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+//! Alice is an AI assistant that can help search the web to answer+//! questions and cite sources. This module takes a trained model+//! and uses it for inference. The training code can be found in another+//! [repository](https://github.com/stractOrg/alice].+//!+//! The model is a hybrid between an RNN and a Transformer. It's therefore+//! possible to save the hidden state and continue the conversation later.+//! To make sure the state has not been tampered with, it is encrypted+//! using an AES-GCM key.+use std::{io::{Read, Write},path::Path,@@ -152,6 +162,7 @@ pub enum ExecutionState {},}+/// A simplified website that the model sees in the prompt.#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]struct ModelWebsite {pub title: String,diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex ad9cbbe8..08d6edb9 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -14,6 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+//! The api module contains the http api and frontend.+//! All http requests are handled using axum.+//! The frontend is served using a combination of axum, askama and astro+//! (with some funky astro hacks to make askama and astro play nice together).+use axum::{body::Body, extract, middleware, routing::get_service, Router};use tokio::sync::Mutex;use tower_http::{compression::CompressionLayer, services::ServeDir};diff --git a/core/src/autosuggest.rs b/core/src/autosuggest.rsindex 12375258..f5ce584b 100644--- a/core/src/autosuggest.rs+++ b/core/src/autosuggest.rs@@ -14,6 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+//! Autosuggest provides the functionality for the little dropdown that appears+//! when you type something into the search bar and queries are suggested.+//! It uses a finite state transducer (fst) to store popular queries+//! and performs a prefix search on the fst to find suggestions.+use fst::{automaton::Str, Automaton, IntoStreamer};use crate::Result;diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 7c3ca0ad..120a9c8c 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -13,6 +13,8 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! The entrypoint module contains all entrypoints that runs the executables.pub mod alice;pub mod autosuggest_scrape;mod centrality;diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 99b76cd6..b00253c0 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -14,6 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+//! The inverted index is the main data structure of the search engine.+//! It is a mapping from terms to a list of documents. Imagine a hash map+//! { term -> \[doc1, doc2, doc3\] } etc. During search, we look up the terms+//! from the query in the index and perform an intersection of the lists of+//! documents. The result is a list of documents that match the query which+//! is then ranked.+//!+//! The inverted index is implemented using tantivy. The inverted index in+//! tantivy is actually a FST (finite state transducer) and not a hash map.+//! This allows us to perform more advanced queries than just term lookups,+//! but the principle is the same.use chrono::NaiveDateTime;use serde::{Deserialize, Serialize};use tantivy::collector::{Collector, Count};diff --git a/core/src/lib.rs b/core/src/lib.rsindex d74deedb..d980368b 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -14,7 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+//! Main library for Stract.// #![warn(clippy::pedantic)]+// #![warn(missing_docs)]+// #![warn(clippy::missing_docs_in_private_items)]#![allow(clippy::cast_precision_loss)]#![allow(clippy::missing_errors_doc)]diff --git a/core/src/main.rs b/core/src/main.rsindex f2a841f7..c5c12bae 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -41,16 +41,18 @@ struct Args {#[derive(Subcommand)]enum Commands {- /// Commands to deploy the Alice server+ /// Commands to deploy the Alice server.Alice {#[clap(subcommand)]options: AliceOptions,},- /// Build the search index++ /// Build the search index.Indexer {#[clap(subcommand)]options: IndexingOptions,},+/// Calculate centrality metrics that estimates a websites importance. These metrics are used to rank search results.Centrality {#[clap(subcommand)]@@ -58,32 +60,40 @@ enum Commands {webgraph_path: String,output_path: String,},+/// Parse the DMOZ dataset. DMOZ contains a list of websites and their categories./// It can be used to calculate the topic centrality for websites or augments website descriptions during indexing.DmozParser {dmoz_file: String,output_path: String,},+/// Webgraph specific commands.Webgraph {#[clap(subcommand)]options: WebgraphOptions,},+/// Deploy the search server.- SearchServer {- config_path: String,- },+ SearchServer { config_path: String },+/// Deploy the frontend. The frontend is a web server that serves the search UI and interacts with/// the search servers, webgraph servers etc. to provide the necesarry functionality.- Frontend {- config_path: String,- },+ Frontend { config_path: String },++ /// Scrape the Google autosuggest API for search queries.AutosuggestScrape {num_queries: usize,gl: Gl,ms_sleep_between_req: u64,output_dir: String,},++ /// Calculate the topic centrality for websites. We use the DMOZ dataset to+ /// determine set of representative websites for each topic. Harmonic centrality approximations+ /// are then calculated for each website in the webgraph.+ ///+ /// This is currently not used in the search engine, so everything related to this might be buggy.TopicCentrality {index_path: String,topics_path: String,@@ -91,12 +101,16 @@ enum Commands {online_harmonic_path: String,output_path: String,},- CrawlCoordinator {- config_path: String,- },- Crawler {- config_path: String,- },++ /// Deploy the crawl coordinator. The crawl coordinator is responsible for+ /// distributing crawl jobs to the crawles and deciding which urls to crawl next.+ CrawlCoordinator { config_path: String },++ /// Deploy the crawler. The crawler is responsible for downloading webpages, saving them to S3,+ /// and sending newly discovered urls back to the crawl coordinator.+ Crawler { config_path: String },++ /// Setup dev environment.#[cfg(feature = "dev")]Configure {#[clap(long, takes_value = false)]@@ -107,9 +121,12 @@ enum Commands {},}+/// Commands to deploy Alice.#[derive(Subcommand)]enum AliceOptions {+ /// Deploy Alice server.Serve { config_path: String },+ /// Generate a new keypair for Alice to sign states.GenerateKey,}diff --git a/core/src/mapreduce/mod.rs b/core/src/mapreduce/mod.rsindex 16808ddf..b9f1ba8a 100644--- a/core/src/mapreduce/mod.rs+++ b/core/src/mapreduce/mod.rs@@ -14,6 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+//! MapReduce is a distributed computing framework for processing large data sets across+//! clusters of computers using a master node to coordinate the work.+use serde::{de::DeserializeOwned, Deserialize, Serialize};mod manager;