Skip to content

web search done right

some module level docs

Mikkel Denkercfe9d0ca79c1
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index df58767c..f9fb7c15 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -14,6 +14,16 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//! Alice is an AI assistant that can help search the web to answer
+//! questions and cite sources. This module takes a trained model
+//! and uses it for inference. The training code can be found in another
+//! [repository](https://github.com/stractOrg/alice].
+//!
+//! The model is a hybrid between an RNN and a Transformer. It's therefore
+//! possible to save the hidden state and continue the conversation later.
+//! To make sure the state has not been tampered with, it is encrypted
+//! using an AES-GCM key.
+
use std::{
io::{Read, Write},
path::Path,
@@ -152,6 +162,7 @@ pub enum ExecutionState {
},
}
+/// A simplified website that the model sees in the prompt.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct ModelWebsite {
pub title: String,
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index ad9cbbe8..08d6edb9 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -14,6 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//! The api module contains the http api and frontend.
+//! All http requests are handled using axum.
+//! The frontend is served using a combination of axum, askama and astro
+//! (with some funky astro hacks to make askama and astro play nice together).
+
use axum::{body::Body, extract, middleware, routing::get_service, Router};
use tokio::sync::Mutex;
use tower_http::{compression::CompressionLayer, services::ServeDir};
diff --git a/core/src/autosuggest.rs b/core/src/autosuggest.rs
index 12375258..f5ce584b 100644
--- a/core/src/autosuggest.rs
+++ b/core/src/autosuggest.rs
@@ -14,6 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//! Autosuggest provides the functionality for the little dropdown that appears
+//! when you type something into the search bar and queries are suggested.
+//! It uses a finite state transducer (fst) to store popular queries
+//! and performs a prefix search on the fst to find suggestions.
+
use fst::{automaton::Str, Automaton, IntoStreamer};
use crate::Result;
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index 7c3ca0ad..120a9c8c 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -13,6 +13,8 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! The entrypoint module contains all entrypoints that runs the executables.
pub mod alice;
pub mod autosuggest_scrape;
mod centrality;
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 99b76cd6..b00253c0 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -14,6 +14,17 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//! The inverted index is the main data structure of the search engine.
+//! It is a mapping from terms to a list of documents. Imagine a hash map
+//! { term -> \[doc1, doc2, doc3\] } etc. During search, we look up the terms
+//! from the query in the index and perform an intersection of the lists of
+//! documents. The result is a list of documents that match the query which
+//! is then ranked.
+//!
+//! The inverted index is implemented using tantivy. The inverted index in
+//! tantivy is actually a FST (finite state transducer) and not a hash map.
+//! This allows us to perform more advanced queries than just term lookups,
+//! but the principle is the same.
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use tantivy::collector::{Collector, Count};
diff --git a/core/src/lib.rs b/core/src/lib.rs
index d74deedb..d980368b 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -14,7 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//! Main library for Stract.
// #![warn(clippy::pedantic)]
+// #![warn(missing_docs)]
+// #![warn(clippy::missing_docs_in_private_items)]
#![allow(clippy::cast_precision_loss)]
#![allow(clippy::missing_errors_doc)]
diff --git a/core/src/main.rs b/core/src/main.rs
index f2a841f7..c5c12bae 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -41,16 +41,18 @@ struct Args {
#[derive(Subcommand)]
enum Commands {
- /// Commands to deploy the Alice server
+ /// Commands to deploy the Alice server.
Alice {
#[clap(subcommand)]
options: AliceOptions,
},
- /// Build the search index
+
+ /// Build the search index.
Indexer {
#[clap(subcommand)]
options: IndexingOptions,
},
+
/// Calculate centrality metrics that estimates a websites importance. These metrics are used to rank search results.
Centrality {
#[clap(subcommand)]
@@ -58,32 +60,40 @@ enum Commands {
webgraph_path: String,
output_path: String,
},
+
/// Parse the DMOZ dataset. DMOZ contains a list of websites and their categories.
/// It can be used to calculate the topic centrality for websites or augments website descriptions during indexing.
DmozParser {
dmoz_file: String,
output_path: String,
},
+
/// Webgraph specific commands.
Webgraph {
#[clap(subcommand)]
options: WebgraphOptions,
},
+
/// Deploy the search server.
- SearchServer {
- config_path: String,
- },
+ SearchServer { config_path: String },
+
/// Deploy the frontend. The frontend is a web server that serves the search UI and interacts with
/// the search servers, webgraph servers etc. to provide the necesarry functionality.
- Frontend {
- config_path: String,
- },
+ Frontend { config_path: String },
+
+ /// Scrape the Google autosuggest API for search queries.
AutosuggestScrape {
num_queries: usize,
gl: Gl,
ms_sleep_between_req: u64,
output_dir: String,
},
+
+ /// Calculate the topic centrality for websites. We use the DMOZ dataset to
+ /// determine set of representative websites for each topic. Harmonic centrality approximations
+ /// are then calculated for each website in the webgraph.
+ ///
+ /// This is currently not used in the search engine, so everything related to this might be buggy.
TopicCentrality {
index_path: String,
topics_path: String,
@@ -91,12 +101,16 @@ enum Commands {
online_harmonic_path: String,
output_path: String,
},
- CrawlCoordinator {
- config_path: String,
- },
- Crawler {
- config_path: String,
- },
+
+ /// Deploy the crawl coordinator. The crawl coordinator is responsible for
+ /// distributing crawl jobs to the crawles and deciding which urls to crawl next.
+ CrawlCoordinator { config_path: String },
+
+ /// Deploy the crawler. The crawler is responsible for downloading webpages, saving them to S3,
+ /// and sending newly discovered urls back to the crawl coordinator.
+ Crawler { config_path: String },
+
+ /// Setup dev environment.
#[cfg(feature = "dev")]
Configure {
#[clap(long, takes_value = false)]
@@ -107,9 +121,12 @@ enum Commands {
},
}
+/// Commands to deploy Alice.
#[derive(Subcommand)]
enum AliceOptions {
+ /// Deploy Alice server.
Serve { config_path: String },
+ /// Generate a new keypair for Alice to sign states.
GenerateKey,
}
diff --git a/core/src/mapreduce/mod.rs b/core/src/mapreduce/mod.rs
index 16808ddf..b9f1ba8a 100644
--- a/core/src/mapreduce/mod.rs
+++ b/core/src/mapreduce/mod.rs
@@ -14,6 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//! MapReduce is a distributed computing framework for processing large data sets across
+//! clusters of computers using a master node to coordinate the work.
+
use serde::{de::DeserializeOwned, Deserialize, Serialize};
mod manager;