Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

13.7 KB
Raw Blame
//! Self-registration with the weed master, mirroring how filer/s3 keep
//! connected: hold a KeepConnected stream open to the leader and the node
//! shows up in ListClusterNodes(client_type="vector").

use std::collections::HashSet;
use std::time::Duration;

use anyhow::{Context, Result};
use tokio::time::sleep;

use crate::master_pb::{
    KeepConnectedRequest, ListClusterNodesRequest, seaweed_client::SeaweedClient,
};
use crate::security::GrpcTls;
pub use weed_lance_worker::cluster::{CONNECT_TIMEOUT, connect, grpc_addr};

fn registration(client_addr: &str, filer_group: &str, metrics_port: u16) -> KeepConnectedRequest {
    KeepConnectedRequest {
        client_type: "vector".into(),
        client_address: client_addr.into(),
        version: env!("CARGO_PKG_VERSION").into(),
        filer_group: filer_group.into(),
        data_center: String::new(),
        rack: String::new(),
        leave_lock_ring: false,
        metrics_port: metrics_port.into(),
    }
}

// `host:http.grpc` normalizes to `host:http` so a leader hint naming the
// dialed master does not read as a redirect.
fn http_addr(master: &str) -> String {
    let rest = master.rsplit_once("://").map(|(_, r)| r).unwrap_or(master);
    match rest.rsplit_once(':') {
        Some((host, port)) => {
            format!("{host}:{}", port.split('.').next().unwrap_or(port))
        }
        None => rest.to_string(),
    }
}

/// Retry forever: try each master until a stream opens, hold it until it
/// errors, then rediscover — same shape as wdclient.KeepConnectedToMaster.
pub async fn keep_connected(
    masters: Vec<String>,
    client_addr: String,
    filer_group: String,
    metrics_port: u16,
    tls: Option<GrpcTls>,
    metrics: crate::metrics::Metrics,
) {
    let registration = &registration(&client_addr, &filer_group, metrics_port);
    let tls = tls.as_ref();
    let metrics = &metrics;
    loop {
        try_all_masters(&masters, |master| async move {
            let result = stream_to(&master, registration, tls, Some(&metrics)).await;
            metrics.set_connected(false).await;
            result
        })
        .await;
        sleep(Duration::from_secs(3)).await;
    }
}

/// One pass, as wdclient.tryAllMasters: a leader hint is followed at once,
/// and one that failed this pass is not followed again, so a follower naming
/// a dead leader cannot hide the other masters.
async fn try_all_masters<F, Fut>(masters: &[String], mut connect: F)
where
    F: FnMut(String) -> Fut,
    Fut: Future<Output = Result<Option<String>>>,
{
    let mut failed = HashSet::new();
    for master in masters {
        if failed.contains(&http_addr(master)) {
            continue;
        }
        let mut target = master.clone();
        // Bounded, so two masters naming each other cannot spin.
        for _ in 0..=masters.len() {
            match connect(target.clone()).await {
                Ok(Some(hint)) if !failed.contains(&http_addr(&hint)) => target = hint,
                Ok(_) => break,
                Err(e) => {
                    tracing::warn!("master {target}: {e:#}");
                    failed.insert(http_addr(&target));
                    break;
                }
            }
        }
    }
}

async fn stream_to(
    master: &str,
    registration: &KeepConnectedRequest,
    tls: Option<&GrpcTls>,
    metrics: Option<&crate::metrics::Metrics>,
) -> Result<Option<String>> {
    let channel = connect(master, tls).await?;
    // A master answers at once, a follower with its leader hint.
    let answer_by = tokio::time::Instant::now() + CONNECT_TIMEOUT;
    let mut client = SeaweedClient::new(channel);
    let (tx, rx) = tokio::sync::mpsc::channel(8);
    tx.send(registration.clone()).await?;
    let mut stream = tokio::time::timeout_at(
        answer_by,
        client.keep_connected(tokio_stream::wrappers::ReceiverStream::new(rx)),
    )
    .await
    .context("master did not answer keep-connected")??
    .into_inner();
    let mut answered = false;
    let interval = Duration::from_secs(5);
    let mut heartbeat = tokio::time::interval_at(tokio::time::Instant::now() + interval, interval);
    // The master keeps the node live while the stream is open; resend the
    // registration periodically so a reconnect never leaves a stale entry.
    loop {
        tokio::select! {
            msg = stream.message() => match msg? {
                Some(resp) => {
                    answered = true;
                    // A volume_location appears on every response — only
                    // follow it when the hint points at a different master,
                    // otherwise the leader keeps identifying itself and we
                    // would reconnect forever.
                    if let Some(loc) = resp.volume_location {
                        if !loc.leader.is_empty()
                            && http_addr(&loc.leader) != http_addr(master)
                        {
                            return Ok(Some(loc.leader));
                        }
                    }
                    if let Some(metrics) = metrics { metrics.set_connected(true).await; }
                }
                None => anyhow::bail!("master closed the keep-connected stream"),
            },
            _ = tokio::time::sleep_until(answer_by), if !answered => {
                anyhow::bail!("master did not answer keep-connected");
            }
            _ = heartbeat.tick() => {
                tokio::time::timeout(CONNECT_TIMEOUT, tx.send(registration.clone())).await
                    .context("registration send stalled")??;
                probe_registration(&mut client).await?;
            }
        }
    }
}

async fn probe_registration(client: &mut SeaweedClient<tonic::transport::Channel>) -> Result<()> {
    tokio::time::timeout(
        CONNECT_TIMEOUT,
        client.list_cluster_nodes(ListClusterNodesRequest {
            client_type: "vector".into(),
            ..Default::default()
        }),
    )
    .await
    .context("registration heartbeat stalled")??;
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn registration_preserves_filer_group() {
        assert_eq!(
            registration("vector:8090", "tenant-a", 0).filer_group,
            "tenant-a"
        );
    }

    #[test]
    fn registration_advertises_metrics_port() {
        assert_eq!(registration("vector:8090", "", 9327).metrics_port, 9327);
        assert_eq!(registration("vector:8090", "", 0).metrics_port, 0);
    }

    #[tokio::test]
    async fn stalled_registration_heartbeat_is_bounded() {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let hold = tokio::spawn(async move {
            let (_connection, _) = listener.accept().await.unwrap();
            std::future::pending::<()>().await;
        });
        let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}"))
            .unwrap()
            .connect_lazy();
        let mut client = SeaweedClient::new(channel);
        let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, probe_registration(&mut client))
            .await
            .unwrap();
        assert!(format!("{:#}", result.unwrap_err()).contains("registration heartbeat stalled"));
        hold.abort();
    }

    /// Runs one pass against scripted masters and returns the dial order.
    async fn pass(masters: &[&str], answer: fn(&str) -> Result<Option<String>>) -> Vec<String> {
        let masters: Vec<String> = masters.iter().map(|m| m.to_string()).collect();
        let mut tried = Vec::new();
        try_all_masters(&masters, |master| {
            tried.push(master.clone());
            async move { answer(&master) }
        })
        .await;
        tried
    }

    #[tokio::test]
    async fn a_dead_leader_hint_does_not_hide_the_other_masters() {
        let tried = pass(&["m1:9333", "m2:9333", "m3:9333"], |master| match master {
            "m1:9333" | "m2:9333" => Ok(Some("dead:9333.19333".to_string())),
            _ => anyhow::bail!("unreachable"),
        })
        .await;
        assert_eq!(tried, ["m1:9333", "dead:9333.19333", "m2:9333", "m3:9333"]);
    }

    #[tokio::test]
    async fn a_hint_naming_a_failed_master_in_grpc_form_is_skipped() {
        let tried = pass(&["m1:9333", "m2:9333"], |master| match master {
            "m1:9333" => anyhow::bail!("unreachable"),
            _ => Ok(Some("m1:9333.19333".to_string())),
        })
        .await;
        assert_eq!(tried, ["m1:9333", "m2:9333"]);
    }

    #[tokio::test]
    async fn masters_naming_each_other_do_not_spin() {
        let tried = pass(&["m1:9333", "m2:9333"], |master| {
            let other = if master == "m1:9333" {
                "m2:9333"
            } else {
                "m1:9333"
            };
            Ok(Some(other.to_string()))
        })
        .await;
        assert_eq!(tried.len(), 6);
    }

    #[tokio::test]
    async fn dial_to_a_black_holed_master_is_bounded() {
        // TEST-NET-1 (RFC 5737) is reserved, so nothing answers the SYN.
        let registration = registration("127.0.0.1:8080", "", 0);
        let dial = stream_to("192.0.2.1:9333", &registration, None, None);
        let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, dial).await;
        assert!(matches!(result, Ok(Err(_))), "dial was not bounded");
    }

    #[tokio::test]
    async fn a_master_that_never_answers_is_bounded() {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let port = listener.local_addr().unwrap().port();
        tokio::spawn(async move {
            let (mut conn, _) = listener.accept().await.unwrap();
            let _ = tokio::io::copy(&mut conn, &mut tokio::io::sink()).await;
        });
        let master = format!("127.0.0.1:9333.{port}");
        let registration = registration("127.0.0.1:8080", "", 0);
        let open = stream_to(&master, &registration, None, None);
        match tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await {
            Ok(Err(e)) => assert!(format!("{e:#}").contains("did not answer"), "{e:#}"),
            other => panic!("open was not bounded: {other:?}"),
        }
    }

    /// The worker crate's `tests/tls`: a CA and the `engine-a` leaf it signed.
    fn credentials() -> GrpcTls {
        let fixture = |name: &str| {
            std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
                .join("../../../seaweed-worker/crates/lance/tests/tls")
                .join(name)
                .display()
                .to_string()
        };
        GrpcTls {
            ca_path: fixture("ca.pem"),
            cert_path: fixture("a.pem"),
            key_path: fixture("a.key"),
            allowed_server_names: Vec::new(),
            allowed_server_wildcard_domain: String::new(),
        }
    }

    #[tokio::test]
    async fn a_master_that_stalls_the_tls_handshake_is_bounded() {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let port = listener.local_addr().unwrap().port();
        tokio::spawn(async move {
            let (mut conn, _) = listener.accept().await.unwrap();
            let _ = tokio::io::copy(&mut conn, &mut tokio::io::sink()).await;
        });
        let master = format!("127.0.0.1:9333.{port}");
        let tls = credentials();
        let registration = registration("127.0.0.1:8080", "", 0);
        let open = stream_to(&master, &registration, Some(&tls), None);
        match tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await {
            Ok(Err(e)) => assert!(format!("{e:#}").contains("did not complete"), "{e:#}"),
            other => panic!("handshake was not bounded: {other:?}"),
        }
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn a_credential_read_that_stalls_is_bounded() {
        let dir = tempfile::tempdir().unwrap();
        let fifo = dir.path().join("ca.pem");
        let made = std::process::Command::new("mkfifo").arg(&fifo).status();
        assert!(made.unwrap().success());
        // Every connection is dropped at once, so a dial that skipped the read
        // fails its handshake instead of timing out.
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let port = listener.local_addr().unwrap().port();
        tokio::spawn(async move {
            while let Ok((conn, _)) = listener.accept().await {
                drop(conn);
            }
        });
        let mut tls = credentials();
        tls.ca_path = fifo.display().to_string();
        let master = format!("127.0.0.1:9333.{port}");
        let registration = registration("127.0.0.1:8080", "", 0);
        let open = stream_to(&master, &registration, Some(&tls), None);
        let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await;
        use std::io::Write;
        let mut cleanup = std::fs::OpenOptions::new()
            .read(true)
            .write(true)
            .open(&fifo)
            .unwrap();
        let replacement = dir.path().join("ca.replacement");
        std::fs::write(&replacement, b"invalid CA").unwrap();
        std::fs::rename(&replacement, &fifo).unwrap();
        cleanup.write_all(b"invalid CA").unwrap();
        drop(cleanup);
        match result {
            Ok(Err(e)) => assert!(format!("{e:#}").contains("did not complete"), "{e:#}"),
            other => panic!("credential read was not bounded: {other:?}"),
        }
    }

    #[test]
    fn grpc_addr_refuses_a_port_past_65535() {
        assert_eq!(grpc_addr("m1:9333").unwrap(), "http://m1:19333");
        assert_eq!(grpc_addr("m1:55535").unwrap(), "http://m1:65535");
        assert_eq!(grpc_addr("m1:60000.7000").unwrap(), "http://m1:7000");
        assert!(grpc_addr("m1:55536").is_err());
        assert!(grpc_addr("https://m1:60000").is_err());
        assert!(grpc_addr("m1:65536").is_err());
        assert!(grpc_addr("m1:9333.65536").is_err());
    }
}