Skip to content

web search done right

4.0 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use std::time::Duration;

pub struct Collector;

impl Collector {
    pub fn site_penalty() -> f64 {
        0.1
    }

    pub fn title_penalty() -> f64 {
        1.0
    }

    pub fn url_penalty() -> f64 {
        20.0
    }

    pub fn url_without_tld_penalty() -> f64 {
        1.0
    }

    pub fn max_docs_considered() -> usize {
        250_000
    }
}

pub struct Api;

impl Api {
    pub fn stackoverflow() -> f64 {
        0.5
    }

    pub fn entity_sidebar() -> f64 {
        10.0
    }

    pub fn max_concurrent_searches() -> Option<usize> {
        None
    }

    pub fn max_similar_hosts() -> usize {
        1_000
    }

    pub fn top_phrases_for_autosuggest() -> usize {
        1_000_000
    }
}

pub struct Snippet;

impl Snippet {
    pub fn desired_num_chars() -> usize {
        275
    }

    pub fn delta_num_chars() -> usize {
        50
    }

    pub fn min_passage_width() -> usize {
        20
    }

    pub fn empty_query_snippet_words() -> usize {
        50
    }

    pub fn min_description_words() -> usize {
        10
    }

    pub fn min_body_length() -> usize {
        256
    }

    pub fn min_body_length_homepage() -> usize {
        1024
    }
}

pub struct Crawler;

impl Crawler {
    pub fn robots_txt_cache_sec() -> u64 {
        60 * 60
    }

    pub fn min_politeness_factor() -> u32 {
        0
    }

    pub fn start_politeness_factor() -> u32 {
        2
    }

    pub fn max_politeness_factor() -> u32 {
        11
    }

    pub fn min_crawl_delay_ms() -> u64 {
        10_000
    }

    pub fn max_crawl_delay_ms() -> u64 {
        180_000
    }

    pub fn max_url_slowdown_retry() -> u8 {
        3
    }

    pub fn timeout_seconds() -> u64 {
        60
    }
}

pub struct SearchQuery;

impl SearchQuery {
    pub fn flatten_response() -> bool {
        true
    }

    pub fn return_ranking_signals() -> bool {
        false
    }

    pub fn safe_search() -> bool {
        false
    }

    pub fn count_results_exact() -> bool {
        false
    }

    pub fn return_structured_data() -> bool {
        false
    }
}

pub struct Widgets;

impl Widgets {
    pub fn calculator_fetch_currencies_exchange() -> bool {
        true
    }
}

pub struct Indexing;

impl Indexing {
    pub fn batch_size() -> usize {
        512
    }

    pub fn autocommit_after_num_inserts() -> usize {
        25_000
    }
}

pub struct ApproxHarmonic;
impl ApproxHarmonic {
    pub fn sample_rate() -> f64 {
        0.3
    }

    pub fn max_distance() -> u8 {
        7
    }
}

pub struct Webgraph;
impl Webgraph {
    pub fn merge_all_segments() -> bool {
        true
    }
}

pub struct LiveCrawler;
impl LiveCrawler {
    pub fn feeds_crawl_interval() -> Duration {
        Duration::from_secs(30 * 60) // 30 minutes
    }

    pub fn sitemap_crawl_interval() -> Duration {
        Duration::from_secs(24 * 60 * 60) // daily
    }

    pub fn frontpage_crawl_interval() -> Duration {
        Duration::from_secs(12 * 60 * 60) // 12 hours
    }

    pub fn blogs_budget() -> u64 {
        300_000
    }

    pub fn news_budget() -> u64 {
        500_000
    }

    pub fn remaining_budget() -> u64 {
        50_000
    }

    pub fn init_crawl_db() -> bool {
        true
    }
}

pub struct HarmonicNearestSeed;
impl HarmonicNearestSeed {
    pub fn discount_factor() -> f64 {
        0.5
    }
}