A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::config::{Config, DeliveryConfig};
use crate::delivery;
use crate::metrics::Metrics;
use crate::relay;
use crate::relay_consent::{self, RelayConsentStore};
use crate::rpc::RpcClient;
use crate::tokens::TokenCache;
use crate::vendor;
use axum::Router;
use axum::extract::ConnectInfo;
use fluxer_common::secret::SecretString;
use fluxer_svc::shutdown::{DEFAULT_DRAIN_TIMEOUT, drain_with_timeout, wait_for_shutdown};
use fluxer_svc::transport::NatsTransport;
use hyper::body::Incoming;
use hyper::service::service_fn;
use hyper_util::rt::{TokioExecutor, TokioIo};
use hyper_util::server::conn::auto::Builder;
use std::io;
use std::net::SocketAddr;
use std::pin::pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{Notify, oneshot, watch};
use tokio::task::JoinHandle;
use tokio::time::{Instant, sleep};
use tower_service::Service;
use tracing::{error, info};
const JOB_DRAIN_TIMEOUT: Duration = Duration::from_secs(25);
const FIRST_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
const CONNECTION_IDLE_TIMEOUT: Duration = Duration::from_secs(120);
const CONNECTION_CLOSE_GRACE: Duration = Duration::from_secs(15);
const ACCEPT_ERROR_PAUSE: Duration = Duration::from_secs(1);
pub struct Sidecar {
metrics: Arc<Metrics>,
serving: Arc<AtomicBool>,
}
impl Sidecar {
pub fn new(metrics: Arc<Metrics>) -> Self {
Self {
metrics,
serving: Arc::new(AtomicBool::new(false)),
}
}
pub fn router(&self) -> Router {
let metrics = self.metrics.clone();
fluxer_svc::server::ops_router(self.serving.clone(), Arc::new(move || metrics.render()))
}
pub fn set_serving(&self, serving: bool) {
self.serving.store(serving, Ordering::SeqCst);
}
}
pub struct Serving {
stop: oneshot::Sender<()>,
served: JoinHandle<()>,
}
impl Serving {
pub async fn stop(self) {
let _ = self.stop.send(());
drain_with_timeout(
async {
let _ = self.served.await;
},
DEFAULT_DRAIN_TIMEOUT,
)
.await;
}
}
pub async fn serve(addr: SocketAddr, router: Router) -> anyhow::Result<Serving> {
let listener = TcpListener::bind(addr).await?;
let (stop, stopped) = oneshot::channel::<()>();
let served = tokio::spawn(accept_connections(listener, router, stopped));
Ok(Serving { stop, served })
}
async fn accept_connections(
listener: TcpListener,
router: Router,
mut stopped: oneshot::Receiver<()>,
) {
let stopping = watch::Sender::new(false);
let (open_tx, open_rx) = watch::channel(());
loop {
let (stream, remote_addr) = tokio::select! {
accepted = listener.accept() => match accepted {
Ok(accepted) => accepted,
Err(err) => {
pause_after_accept_error(err).await;
continue;
}
},
_ = &mut stopped => break,
};
tokio::spawn(serve_connection(
stream,
remote_addr,
router.clone(),
stopping.subscribe(),
open_rx.clone(),
));
}
stopping.send_replace(true);
drop(open_rx);
drop(listener);
open_tx.closed().await;
}
async fn pause_after_accept_error(err: io::Error) {
if matches!(
err.kind(),
io::ErrorKind::ConnectionRefused
| io::ErrorKind::ConnectionAborted
| io::ErrorKind::ConnectionReset
) {
return;
}
error!(error = %err, "accept error");
sleep(ACCEPT_ERROR_PAUSE).await;
}
async fn serve_connection(
stream: TcpStream,
remote_addr: SocketAddr,
router: Router,
mut stopping: watch::Receiver<bool>,
_open: watch::Receiver<()>,
) {
let activity = Arc::new(Notify::new());
let service = service_fn({
let activity = Arc::clone(&activity);
move |mut req: hyper::Request<Incoming>| {
activity.notify_one();
req.extensions_mut().insert(ConnectInfo(remote_addr));
router.clone().call(req)
}
});
let mut builder = Builder::new(TokioExecutor::new());
builder.http2().enable_connect_protocol();
let mut conn = pin!(builder.serve_connection_with_upgrades(TokioIo::new(stream), service));
let mut deadline = pin!(sleep(FIRST_REQUEST_TIMEOUT));
let mut served_request = false;
let mut closing = false;
loop {
tokio::select! {
_ = conn.as_mut() => break,
_ = activity.notified(), if !closing => {
served_request = true;
deadline.as_mut().reset(Instant::now() + CONNECTION_IDLE_TIMEOUT);
}
_ = stopping.wait_for(|stopping| *stopping), if !closing => {
conn.as_mut().graceful_shutdown();
closing = true;
deadline.as_mut().reset(Instant::now() + CONNECTION_CLOSE_GRACE);
}
_ = deadline.as_mut() => {
if closing || !served_request {
break;
}
conn.as_mut().graceful_shutdown();
closing = true;
deadline.as_mut().reset(Instant::now() + CONNECTION_CLOSE_GRACE);
}
}
}
}
pub struct AppState {
pub(crate) cfg: DeliveryConfig,
pub(crate) metrics: Arc<Metrics>,
pub(crate) sidecar: Arc<Sidecar>,
pub(crate) relay_consent: RelayConsentStore,
pub(crate) rpc: RpcClient,
pub(crate) http: reqwest::Client,
pub(crate) web_push_http: reqwest::Client,
pub(crate) apns_http: reqwest::Client,
pub(crate) tokens: TokenCache,
pub(crate) draining: watch::Sender<bool>,
}
impl AppState {
pub(crate) fn try_new(cfg: DeliveryConfig) -> anyhow::Result<Self> {
let metrics = Arc::new(Metrics::new());
let relay_consent = RelayConsentStore::new(cfg.relay_consent_accepted);
metrics.record_relay_consent_accepted(relay_consent.accepted());
let http = vendor::http_client()?;
Ok(Self {
rpc: RpcClient::new(&cfg.rpc, http.clone(), Arc::clone(&metrics)),
relay_consent,
sidecar: Arc::new(Sidecar::new(Arc::clone(&metrics))),
web_push_http: vendor::web_push_http_client(&cfg.private_hosts)?,
apns_http: vendor::apns_http_client()?,
tokens: TokenCache::new(),
draining: watch::Sender::new(false),
cfg,
metrics,
http,
})
}
}
pub async fn run(cfg: Config) -> anyhow::Result<()> {
match cfg {
Config::Delivery(cfg) => run_delivery(*cfg).await,
Config::Relay(cfg) => relay::run(*cfg).await,
}
}
async fn run_delivery(cfg: DeliveryConfig) -> anyhow::Result<()> {
let state = Arc::new(AppState::try_new(cfg)?);
let addr = state.cfg.bind_addr;
let serving = serve(addr, state.sidecar.router()).await?;
info!(%addr, vapid_subject = state.cfg.vapid.email, "push sidecar listening");
let transport = NatsTransport::connect(
&state.cfg.nats.url,
state.cfg.nats.auth_token.as_ref().map(SecretString::expose),
)
.await?;
state.sidecar.set_serving(true);
let subscriber = tokio::spawn({
let transport = transport.clone();
let state = Arc::clone(&state);
async move {
relay_consent::run_subscriber(
transport,
&state.rpc,
&state.relay_consent,
&state.metrics,
relay_consent::RECONCILE_INTERVAL,
)
.await
}
});
let jobs = tokio::spawn(delivery::run_job_subscribers(transport, Arc::clone(&state)));
wait_for_shutdown().await;
state.sidecar.set_serving(false);
subscriber.abort();
drain_jobs(&state, jobs, JOB_DRAIN_TIMEOUT).await;
serving.stop().await;
Ok(())
}
async fn drain_jobs(state: &AppState, jobs: JoinHandle<()>, timeout: Duration) {
state.draining.send_replace(true);
let running = jobs.abort_handle();
drain_with_timeout(
async {
let _ = jobs.await;
},
timeout,
)
.await;
running.abort();
}