Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.6 KB
Raw Blame
use std::{sync::Arc, time::Duration};

use anyhow::Error;
use async_trait::async_trait;
use reqwest::Client;
use tracing::warn;

use super::DataSource;

pub struct UrlList {
    pub urls: Vec<String>,
    pub client: Client,
    pub retries: usize,
}

impl UrlList {
    pub async fn new(
        urls: Vec<String>,
        allow_internal_ips: bool,
        timeout: Duration,
        retries: usize,
        validate_urls: bool,
    ) -> Result<Self, Error> {
        let resolver = Arc::new(common_dns::PublicIPv4Resolver {});

        let mut client = reqwest::Client::builder().timeout(timeout);

        if !allow_internal_ips {
            client = client.dns_resolver(resolver);
        }

        let client = client.build()?;

        let source = Self {
            urls,
            client,
            retries,
        };

        // Validate the passed urls, and assert they all support range requests
        if validate_urls {
            for url in &source.urls {
                source.assert_valid_url(url).await?;
            }
        }

        Ok(source)
    }

    // A url is valid to us if it supports range requests and returns a content-length header
    async fn assert_valid_url(&self, url: &str) -> Result<(), Error> {
        let response = self
            .client
            .head(url)
            .send()
            .await?
            .error_for_status()
            .map_err(|e| Error::msg(format!("Failed to get headers for {url}: {e}")))?;

        let accept_ranges = response
            .headers()
            .get("accept-ranges")
            .ok_or(Error::msg("Missing Accept-Ranges header"))?
            .to_str()
            .map_err(|e| Error::msg(format!("Failed to parse Accept-Ranges header: {e}")))?;

        if accept_ranges != "bytes" {
            return Err(Error::msg(format!(
                "Server does not support range requests for {url}"
            )));
        }

        let content_lenth = response
            .headers()
            .get("content-length")
            .ok_or(Error::msg("Missing Content-Length header"))?
            .to_str()
            .map_err(|e| Error::msg(format!("Failed to parse Content-Length header: {e}")))?;

        content_lenth
            .parse::<u64>()
            .map_err(|e| Error::msg(format!("Failed to parse Content-Length as u64: {e}")))?;

        Ok(())
    }

    async fn get_chunk_inner(&self, key: &str, offset: u64, size: u64) -> Result<Vec<u8>, Error> {
        // Ensure the passed key is in our list of URLs
        if !self.urls.contains(&key.to_string()) {
            return Err(Error::msg("Key not found"));
        }

        let response = self
            .client
            .get(key)
            .header("Range", format!("bytes={}-{}", offset, offset + size - 1))
            .send()
            .await?
            .error_for_status();

        match response {
            Ok(response) => Ok(response.bytes().await.map(|bytes| bytes.to_vec())?),
            Err(e) => Err(e.into()),
        }
    }
}

#[async_trait]
impl DataSource for UrlList {
    async fn keys(&self) -> Result<Vec<String>, Error> {
        Ok(self.urls.clone())
    }

    async fn size(&self, key: &str) -> Result<Option<u64>, Error> {
        // Ensure the passed key is in our list of URLs
        if !self.urls.contains(&key.to_string()) {
            return Err(Error::msg("Key not found"));
        }

        // For some reason calling `content_length()` doesn't work properly here, so we don't do that
        self.client
            .head(key)
            .send()
            .await?
            .headers()
            .get("content-length")
            .ok_or(Error::msg(format!(
                "Could not get content length for {key}"
            )))
            .and_then(|header| {
                header
                    .to_str()
                    .map_err(|e| Error::msg(format!("Failed to parse content length: {e}")))
            })
            .and_then(|length| {
                length
                    .parse::<u64>()
                    .map_err(|e| Error::msg(format!("Failed to parse content length as u64: {e}")))
            })
            .map(Some)
    }

    async fn get_chunk(&self, key: &str, offset: u64, size: u64) -> Result<Vec<u8>, Error> {
        let mut retries = self.retries;
        loop {
            match self.get_chunk_inner(key, offset, size).await {
                Ok(chunk) => return Ok(chunk),
                Err(e) => {
                    if retries == 0 {
                        return Err(e);
                    }
                    warn!(
                        "Encountered error when fetching chunk: {e:?}, remaining retries: {retries}"
                    );
                    tokio::time::sleep(Duration::from_secs(1)).await;
                    retries -= 1;
                }
            }
        }
    }
}

#[cfg(test)]
mod test {

    use std::time::Duration;

    use httpmock::MockServer;

    use crate::source::{url_list::UrlList, DataSource};

    const TEST_CONTENTS: &str = include_str!("../../tests/capture_request_dump.jsonl");

    #[tokio::test]
    async fn test_url_list_creation() {
        let server = MockServer::start();
        let head = server.mock(|when, then| {
            when.method(httpmock::Method::HEAD);
            then.status(200)
                .header("accept-ranges", "bytes")
                .header("content-length", TEST_CONTENTS.len().to_string());
        });

        let urls: Vec<_> = ["/1", "/2"].iter().map(|&path| server.url(path)).collect();
        let url_count = urls.len();
        let source = UrlList::new(urls, true, Duration::from_secs(10), 1, true)
            .await
            .unwrap();
        let keys = source.keys().await.unwrap();
        println!("{keys:?}");

        assert_eq!(head.hits(), url_count);
    }

    #[tokio::test]
    async fn test_url_list_no_accept_ranges() {
        let server = MockServer::start();
        let _ = server.mock(|when, then| {
            when.method(httpmock::Method::HEAD);
            then.status(200)
                .header("accept-ranges", "none")
                .header("content-length", TEST_CONTENTS.len().to_string());
        });

        let urls: Vec<_> = ["/1", "/2"].iter().map(|&path| server.url(path)).collect();
        let source_res = UrlList::new(urls, true, Duration::from_secs(10), 0, true).await;

        assert!(source_res.is_err());
    }

    #[tokio::test]
    async fn test_url_list_missing_accept_ranges() {
        let server = MockServer::start();
        let _ = server.mock(|when, then| {
            when.method(httpmock::Method::HEAD);
            then.status(200)
                .header("content-length", TEST_CONTENTS.len().to_string());
        });

        let urls: Vec<_> = ["/1", "/2"].iter().map(|&path| server.url(path)).collect();
        let source_res = UrlList::new(urls, true, Duration::from_secs(10), 0, true).await;

        assert!(source_res.is_err());
    }

    #[tokio::test]
    async fn test_url_list_missing_content_length() {
        let server = MockServer::start();
        let _ = server.mock(|when, then| {
            when.method(httpmock::Method::HEAD);
            then.status(200).header("accept-ranges", "bytes");
        });

        let urls: Vec<_> = ["/1", "/2"].iter().map(|&path| server.url(path)).collect();
        let source_res = UrlList::new(urls, true, Duration::from_secs(10), 0, true).await;

        assert!(source_res.is_err());
    }

    #[tokio::test]
    async fn test_non_usize_content_length() {
        let server = MockServer::start();
        let _ = server.mock(|when, then| {
            when.method(httpmock::Method::HEAD);
            then.status(200)
                .header("accept-ranges", "bytes")
                .header("content-length", "not a number");
        });

        let urls: Vec<_> = ["/1", "/2"].iter().map(|&path| server.url(path)).collect();
        let source_res = UrlList::new(urls, true, Duration::from_secs(10), 0, true).await;

        assert!(source_res.is_err());
    }

    #[tokio::test]
    async fn test_size() {
        let server = MockServer::start();
        let _ = server.mock(|when, then| {
            when.method(httpmock::Method::HEAD);
            then.status(200)
                .header("accept-ranges", "bytes")
                .header("content-length", TEST_CONTENTS.len().to_string());
        });

        let urls: Vec<_> = ["/1", "/2"].iter().map(|&path| server.url(path)).collect();
        let source = UrlList::new(urls.clone(), true, Duration::from_secs(10), 0, true)
            .await
            .unwrap();
        let size = source.size(&urls[0]).await.unwrap();

        assert_eq!(size, Some(TEST_CONTENTS.len() as u64));
    }

    #[tokio::test]
    async fn test_get_first_100_bytes() {
        let server = MockServer::start();
        let _ = server.mock(|when, then| {
            when.method(httpmock::Method::HEAD);
            then.status(200)
                .header("accept-ranges", "bytes")
                .header("content-length", TEST_CONTENTS.len().to_string());
        });
        let _ = server.mock(|when, then| {
            when.method(httpmock::Method::GET);
            then.status(200)
                .header("accept-ranges", "bytes")
                .header("content-length", 100.to_string())
                .body(&TEST_CONTENTS[0..100]);
        });

        let urls: Vec<_> = ["/1", "/2"].iter().map(|&path| server.url(path)).collect();
        let source = UrlList::new(urls.clone(), true, Duration::from_secs(10), 0, true)
            .await
            .unwrap();
        let chunk = source.get_chunk(&urls[0], 0, 100).await.unwrap();

        assert_eq!(chunk.len(), 100);
        assert_eq!(&chunk, &TEST_CONTENTS.as_bytes()[0..100]);
    }
}