Skip to content

web search done right

8.8 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::Result;
use std::{
    fs,
    future::Future,
    io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
    path::{Path, PathBuf},
};

const CHUNK_SIZE_BYTES: usize = 1024 * 1024; // 1 MB

pub trait Stepper {
    fn step(&self, req: Request) -> impl Future<Output = Response>;
}

impl<F> Stepper for F
where
    F: Fn(Request) -> Response,
{
    async fn step(&self, req: Request) -> Response {
        self(req)
    }
}

#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub enum Request {
    Overview { root: PathBuf },
    Chunk { path: PathBuf, offset: u64 },
}

impl Request {
    pub async fn download<P1, P2, S>(remote_path: P1, to: P2, stepper: &S)
    where
        P1: AsRef<Path>,
        P2: AsRef<Path>,
        S: Stepper,
    {
        let Response::Overview { paths } = stepper
            .step(Self::Overview {
                root: remote_path.as_ref().to_path_buf(),
            })
            .await
        else {
            panic!("unexpected response to request")
        };

        for path in paths {
            let local = to.as_ref().join(path.clone());
            if let Some(parent) = local.parent() {
                if !parent.exists() {
                    fs::create_dir_all(parent).expect("failed to create directory");
                }
            }

            Self::download_file(remote_path.as_ref().join(path.clone()), local, stepper).await;
        }
    }

    pub async fn download_file<P1, P2, S>(remote: P1, local: P2, stepper: &S)
    where
        P1: AsRef<Path>,
        P2: AsRef<Path>,
        S: Stepper,
    {
        let mut offset = 0;
        let file = std::fs::OpenOptions::new()
            .create(true)
            .write(true)
            .truncate(true)
            .open(local.as_ref())
            .unwrap_or_else(|_| panic!("failed to create file {:?}", local.as_ref().as_os_str()));
        let mut writer = BufWriter::new(file);

        loop {
            let Response::Chunk { bytes } = stepper
                .step(Self::Chunk {
                    path: remote.as_ref().to_path_buf(),
                    offset,
                })
                .await
            else {
                panic!("unexpected response to request");
            };

            writer.write_all(&bytes).unwrap();

            if bytes.len() < CHUNK_SIZE_BYTES {
                break;
            }

            offset += bytes.len() as u64;
        }
    }
}

#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub enum Response {
    Overview { paths: Vec<PathBuf> },
    Chunk { bytes: Vec<u8> },
}

impl Response {
    pub fn handle(req: Request) -> Result<Self> {
        match req {
            Request::Overview { root } => Ok(Response::Overview {
                paths: files(root.clone())
                    .into_iter()
                    .filter_map(|p| p.strip_prefix(&root).ok().map(|p| p.to_path_buf()))
                    .collect(),
            }),
            Request::Chunk { path, offset } => {
                let file = fs::File::open(path)?;
                let mut reader = BufReader::new(file);

                reader.seek(SeekFrom::Start(offset)).ok();

                let mut res = Vec::with_capacity(CHUNK_SIZE_BYTES);
                let mut buf = [0; 512];

                while res.len() < CHUNK_SIZE_BYTES {
                    let n = reader.read(&mut buf)?;

                    if n == 0 {
                        break;
                    }

                    res.extend_from_slice(&buf[..n]);
                }

                Ok(Response::Chunk { bytes: res })
            }
        }
    }
}

fn files(root: PathBuf) -> Vec<PathBuf> {
    let mut stack = vec![root];
    let mut res = Vec::new();

    while let Some(path) = stack.pop() {
        if path.is_file() {
            res.push(path);
        } else if let Ok(dir) = fs::read_dir(path) {
            for entry in dir.flatten() {
                stack.push(entry.path());
            }
        }
    }

    res
}

#[cfg(test)]
mod tests {
    use std::path::Path;

    use file_store::gen_temp_dir;

    use super::*;

    fn local_step(req: Request) -> Response {
        Response::handle(req).unwrap()
    }

    async fn download<P1: AsRef<Path>, P2: AsRef<Path>>(remote: P1, local: P2) {
        Request::download(remote, local, &local_step).await
    }

    #[tokio::test]
    async fn test_directory() {
        let a = gen_temp_dir().unwrap();
        std::fs::create_dir_all(a.as_ref().join("test")).unwrap();
        let contents = "this is a test";
        std::fs::write(a.as_ref().join("test").join("file.txt"), contents).unwrap();

        let b = gen_temp_dir().unwrap();
        download(&a, &b).await;

        assert!(b.as_ref().join("test").exists());
        assert!(b.as_ref().join("test").join("file.txt").exists());

        let res = std::fs::read_to_string(b.as_ref().join("test").join("file.txt")).unwrap();

        assert_eq!(contents, res);
    }

    #[tokio::test]
    async fn test_single_file() {
        let a = gen_temp_dir().unwrap();
        std::fs::create_dir_all(&a).unwrap();
        let contents = "this is a test";
        std::fs::write(a.as_ref().join("file.txt"), contents).unwrap();

        let b = gen_temp_dir().unwrap();
        download(&a, &b).await;

        assert!(b.as_ref().join("file.txt").exists());

        let res = std::fs::read_to_string(b.as_ref().join("file.txt")).unwrap();

        assert_eq!(contents, res);
    }

    #[tokio::test]
    async fn test_overwrite() {
        let a = gen_temp_dir().unwrap();
        std::fs::create_dir_all(&a).unwrap();
        let contents = "this is a test";
        std::fs::write(a.as_ref().join("file.txt"), contents).unwrap();

        let b = gen_temp_dir().unwrap();
        std::fs::create_dir_all(&b).unwrap();
        std::fs::write(b.as_ref().join("file.txt"), "this is another test").unwrap();
        download(&a, &b).await;

        assert!(b.as_ref().join("file.txt").exists());

        let res = std::fs::read_to_string(b.as_ref().join("file.txt")).unwrap();

        assert_eq!(contents, res);
    }

    #[tokio::test]
    async fn test_keep_non_copied() {
        let a = gen_temp_dir().unwrap();
        std::fs::create_dir_all(a.as_ref().join("test")).unwrap();
        let contents = "this is a test";
        std::fs::write(a.as_ref().join("test").join("a.txt"), contents).unwrap();

        let b = gen_temp_dir().unwrap();
        std::fs::create_dir_all(b.as_ref().join("test")).unwrap();
        std::fs::write(b.as_ref().join("test").join("b.txt"), contents).unwrap();

        download(&a, &b).await;

        assert!(b.as_ref().join("test").exists());
        assert!(b.as_ref().join("test").join("a.txt").exists());
        assert!(b.as_ref().join("test").join("b.txt").exists());

        let res = std::fs::read_to_string(b.as_ref().join("test").join("a.txt")).unwrap();
        assert_eq!(contents, res);

        let res = std::fs::read_to_string(b.as_ref().join("test").join("b.txt")).unwrap();
        assert_eq!(contents, res);
    }

    #[tokio::test]
    async fn test_file_size_edge_case() {
        let content = "a".repeat(CHUNK_SIZE_BYTES - 1);
        let a = gen_temp_dir().unwrap();
        std::fs::create_dir_all(&a).unwrap();
        std::fs::write(a.as_ref().join("minus_1.txt"), &content).unwrap();
        std::fs::write(a.as_ref().join("edge.txt"), format!("{}a", &content)).unwrap();
        std::fs::write(a.as_ref().join("plus_1.txt"), format!("{}aa", &content)).unwrap();

        let b = gen_temp_dir().unwrap();
        download(&a, &b).await;
        assert!(b.as_ref().join("minus_1.txt").exists());
        assert!(b.as_ref().join("edge.txt").exists());
        assert!(b.as_ref().join("plus_1.txt").exists());

        let res = std::fs::read_to_string(b.as_ref().join("minus_1.txt")).unwrap();
        assert_eq!(content, res);

        let res = std::fs::read_to_string(b.as_ref().join("edge.txt")).unwrap();
        assert_eq!(format!("{}a", &content), res);

        let res = std::fs::read_to_string(b.as_ref().join("plus_1.txt")).unwrap();
        assert_eq!(format!("{}aa", &content), res);

        std::fs::remove_dir_all(&a).unwrap();
        std::fs::remove_dir_all(&b).unwrap();
    }
}