Skip to content

web search done right

4.8 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use std::str::FromStr;

use crate::{
    enum_map::{EnumSet, InsertEnumMapKey},
    Error, Result,
};

use super::Html;

#[derive(Debug)]
pub enum RobotsMeta {
    NoIndex,
    NoFollow,
}

impl FromStr for RobotsMeta {
    type Err = anyhow::Error;

    fn from_str(s: &str) -> Result<Self> {
        match s {
            "noindex" => Ok(RobotsMeta::NoIndex),
            "nofollow" => Ok(RobotsMeta::NoFollow),
            _ => Err(Error::UnknownRobotsMetaTag.into()),
        }
    }
}

impl InsertEnumMapKey for RobotsMeta {
    fn into_usize(self) -> usize {
        match self {
            RobotsMeta::NoIndex => 0,
            RobotsMeta::NoFollow => 1,
        }
    }
}

impl Html {
    pub fn parse_robots_meta(&self) -> Option<EnumSet<RobotsMeta>> {
        let mut robots = EnumSet::new();

        for node in self.root.select("meta").unwrap() {
            if let Some(element) = node.as_node().as_element() {
                if let Some(name) = element.attributes.borrow().get("name") {
                    if name == "robots" {
                        if let Some(content) = element.attributes.borrow().get("content") {
                            for part in content.split(',') {
                                let part = part.trim();
                                if let Ok(meta) = part.parse::<RobotsMeta>() {
                                    robots.insert(meta);
                                }
                            }
                        }
                    }
                }
            }
        }

        if robots.is_empty() {
            None
        } else {
            Some(robots)
        }
    }

    pub fn is_no_index(&self) -> bool {
        self.robots
            .as_ref()
            .map(|robots| robots.contains(RobotsMeta::NoIndex))
            .unwrap_or(false)
    }

    pub fn is_no_follow(&self) -> bool {
        self.robots
            .as_ref()
            .map(|robots| robots.contains(RobotsMeta::NoFollow))
            .unwrap_or(false)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn robots_meta_tag() {
        let html = Html::parse(
            r#"
            <html>
                <head>
                    <meta name="robots" content="noindex, nofollow" />
                </head>
                <body>
                </body>
            </html>
        "#,
            "https://www.example.com/whatever",
        )
        .unwrap();

        assert!(html.is_no_index());
        assert!(html.is_no_follow());

        let html = Html::parse(
            r#"
            <html>
                <head>
                    <meta name="robots" content="noindex,nofollow" />
                </head>
                <body>
                </body>
            </html>
        "#,
            "https://www.example.com/whatever",
        )
        .unwrap();

        assert!(html.is_no_index());
        assert!(html.is_no_follow());

        let html = Html::parse(
            r#"
            <html>
                <head>
                    <meta name="robots" content="noindex" />
                </head>
                <body>
                </body>
            </html>
        "#,
            "https://www.example.com/whatever",
        )
        .unwrap();

        assert!(html.is_no_index());
        assert!(!html.is_no_follow());

        let html = Html::parse(
            r#"
            <html>
                <head>
                    <meta name="robots" content="nofollow" />
                </head>
                <body>
                </body>
            </html>
        "#,
            "https://www.example.com/whatever",
        )
        .unwrap();

        assert!(!html.is_no_index());
        assert!(html.is_no_follow());

        let html = Html::parse(
            r#"
            <html>
                <head>
                </head>
                <body>
                </body>
            </html>
        "#,
            "https://www.example.com/whatever",
        )
        .unwrap();

        assert!(!html.is_no_index());
        assert!(!html.is_no_follow());
    }
}