Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

fix(server): a caller holds nothing until it is in, weak tokens are refused, wrong ones wait, and a stop cancels first (#337)

Jareer363800f3b049
diff --git a/src/crates/concat-api/src/job.rs b/src/crates/concat-api/src/job.rs
new file mode 100644
index 00000000..cd01ed46
--- /dev/null
+++ b/src/crates/concat-api/src/job.rs
@@ -0,0 +1,223 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! What the API remembers about its jobs: how far each running one has
+//! got, how each finished one ended, and the flag that stops it.
+//!
+//! Events are fire-and-forget: a caller that was not listening when an
+//! export ended - a script that reconnected, an agent that asks in turns
+//! rather than holding a stream open - would otherwise never learn how
+//! it went. The book is fed by the same events every caller is sent, so
+//! [`crate::Request::ExportStatus`] and the event stream cannot disagree.
+//!
+//! It is also how a server that is stopping cancels the API's own jobs
+//! without touching anybody else's: the export slot can be shared with a
+//! window, so "cancel whatever is in the slot" could stop the window's
+//! export; the flags kept here are only the API's.
+
+use std::collections::{BTreeMap, VecDeque};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+
+use crate::message::{ApiError, ErrorCode, Event, JobState, JobStatus};
+
+/// How many finished jobs are remembered, oldest forgotten first.
+const FINISHED_KEPT: usize = 32;
+
+/// The running jobs' flags and every job's last known state.
+#[derive(Default)]
+pub(crate) struct Book {
+ inner: Mutex<Inner>,
+}
+
+#[derive(Default)]
+struct Inner {
+ /// Each running job's stop flag, by name.
+ running: BTreeMap<String, Arc<AtomicBool>>,
+ /// Running jobs' states, by name.
+ states: BTreeMap<String, JobStatus>,
+ /// Finished jobs, in the order they finished.
+ finished: VecDeque<JobStatus>,
+}
+
+impl Book {
+ fn inner(&self) -> std::sync::MutexGuard<'_, Inner> {
+ self.inner
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ }
+
+ /// A job has begun, stopped by `cancel`.
+ pub(crate) fn start(&self, status: JobStatus, cancel: Arc<AtomicBool>) {
+ let mut inner = self.inner();
+ inner.running.insert(status.job.clone(), cancel);
+ inner.states.insert(status.job.clone(), status);
+ }
+
+ /// Folds one event into its job's state. The last event of a job
+ /// moves it to the finished list and lets its flag go.
+ pub(crate) fn record(&self, event: &Event) {
+ let mut inner = self.inner();
+ let job = event.job().to_owned();
+ let Some(status) = inner.states.get_mut(&job) else {
+ return;
+ };
+ match event {
+ Event::CutoutProgress { fraction, .. } => {
+ status.stage = Some("cutout".to_owned());
+ status.fraction = f64::from(*fraction).clamp(0.0, 1.0);
+ }
+ Event::ExportProgress {
+ frame,
+ total,
+ stage,
+ ..
+ } => {
+ status.stage = Some(stage.clone());
+ status.frame = *frame;
+ status.total = *total;
+ status.fraction = if *total > 0 {
+ (*frame as f64 / *total as f64).clamp(0.0, 1.0)
+ } else {
+ 0.0
+ };
+ }
+ Event::ExportDone {
+ output,
+ width,
+ height,
+ ..
+ } => {
+ status.state = JobState::Done;
+ status.output = output.clone();
+ status.width = Some(*width);
+ status.height = Some(*height);
+ status.fraction = 1.0;
+ }
+ Event::ExportFailed { error, .. } => {
+ status.state = if error.code == ErrorCode::Cancelled {
+ JobState::Cancelled
+ } else {
+ JobState::Failed
+ };
+ status.error = Some(error.clone());
+ }
+ }
+ if status.state != JobState::Running {
+ inner.running.remove(&job);
+ if let Some(done) = inner.states.remove(&job) {
+ inner.finished.push_back(done);
+ while inner.finished.len() > FINISHED_KEPT {
+ inner.finished.pop_front();
+ }
+ }
+ }
+ }
+
+ /// A job's thread ended without saying how - a panic in the render.
+ /// Recorded as failed so a caller asking is not told "running" forever.
+ pub(crate) fn abandon(&self, job: &str) {
+ let running = self.inner().states.contains_key(job);
+ if running {
+ self.record(&Event::ExportFailed {
+ job: job.to_owned(),
+ path: String::new(),
+ error: ApiError::failed("the job ended without a result"),
+ });
+ }
+ }
+
+ /// The state of `job`, running or finished.
+ pub(crate) fn status(&self, job: &str) -> Option<JobStatus> {
+ let inner = self.inner();
+ inner.states.get(job).cloned().or_else(|| {
+ inner
+ .finished
+ .iter()
+ .rev()
+ .find(|status| status.job == job)
+ .cloned()
+ })
+ }
+
+ /// Raises every running job's stop flag. How many there were.
+ pub(crate) fn cancel_all(&self) -> usize {
+ let inner = self.inner();
+ for flag in inner.running.values() {
+ flag.store(true, Ordering::Relaxed);
+ }
+ inner.running.len()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn begun(book: &Book, job: &str) -> Arc<AtomicBool> {
+ let flag = Arc::new(AtomicBool::new(false));
+ book.start(JobStatus::running(job, "/p", "/o.mp4"), Arc::clone(&flag));
+ flag
+ }
+
+ #[test]
+ fn a_job_is_followed_from_progress_to_its_end() {
+ let book = Book::default();
+ begun(&book, "j1");
+ book.record(&Event::ExportProgress {
+ job: "j1".to_owned(),
+ path: "/p".to_owned(),
+ frame: 30,
+ total: 120,
+ stage: "video".to_owned(),
+ });
+ let running = book.status("j1").expect("known");
+ assert_eq!(running.state, JobState::Running);
+ assert_eq!((running.frame, running.total), (30, 120));
+ assert!((running.fraction - 0.25).abs() < 1e-9);
+ book.record(&Event::ExportDone {
+ job: "j1".to_owned(),
+ path: "/p".to_owned(),
+ output: "/o.mp4".to_owned(),
+ width: 16,
+ height: 9,
+ });
+ let done = book.status("j1").expect("still known");
+ assert_eq!(done.state, JobState::Done);
+ assert_eq!((done.width, done.height), (Some(16), Some(9)));
+ assert_eq!(book.cancel_all(), 0, "a finished job has no flag to raise");
+ assert!(book.status("j2").is_none());
+ }
+
+ #[test]
+ fn cancel_all_raises_only_the_running_flags_and_a_cancel_is_its_own_state() {
+ let book = Book::default();
+ let flag = begun(&book, "j1");
+ assert_eq!(book.cancel_all(), 1);
+ assert!(flag.load(Ordering::Relaxed));
+ book.record(&Event::ExportFailed {
+ job: "j1".to_owned(),
+ path: "/p".to_owned(),
+ error: ApiError::new(ErrorCode::Cancelled, "The export was cancelled"),
+ });
+ assert_eq!(
+ book.status("j1").expect("known").state,
+ JobState::Cancelled
+ );
+ }
+
+ #[test]
+ fn a_job_that_vanished_is_failed_and_old_ones_are_forgotten() {
+ let book = Book::default();
+ begun(&book, "j0");
+ book.abandon("j0");
+ assert_eq!(book.status("j0").expect("known").state, JobState::Failed);
+ for index in 1..=FINISHED_KEPT {
+ let job = format!("j{index}");
+ begun(&book, &job);
+ book.abandon(&job);
+ }
+ assert!(book.status("j0").is_none(), "the oldest is forgotten");
+ assert!(book.status(&format!("j{FINISHED_KEPT}")).is_some());
+ }
+}
diff --git a/src/crates/concat-api/src/lib.rs b/src/crates/concat-api/src/lib.rs
index 0845a3dc..a23bd9f8 100644
--- a/src/crates/concat-api/src/lib.rs
+++ b/src/crates/concat-api/src/lib.rs
@@ -28,6 +28,7 @@
//! once and the job reports through the [`EventSink`] the API was made
//! with, from that thread, which is why the sink must be `Send + Sync`.
+mod job;
pub mod message;
pub mod rpc;
@@ -52,8 +53,8 @@ use concat_project::Command;
use concat_project::model::VideoSettings;
pub use message::{
- API_VERSION, ApiError, Dirs, Done, ErrorCode, Event, ExportSpec, Fill, PackageInfo, ParamInfo,
- Picture, Reply, Request, Response, Started, VersionInfo, Written,
+ API_VERSION, ApiError, Dirs, Done, ErrorCode, Event, ExportSpec, Fill, JobState, JobStatus,
+ PackageInfo, ParamInfo, Picture, Reply, Request, Response, Started, VersionInfo, Written,
};
/// The export sheet's middle quality, and what an export gets when the
@@ -162,10 +163,38 @@ pub struct Api {
exporter: Exporter,
events: EventSink,
jobs: Jobs,
+ /// Every job's last known state and each running one's stop flag, fed
+ /// by the events on their way to `events`.
+ book: Arc<job::Book>,
/// [`VersionInfo::capabilities`], in the order they were added.
capabilities: Vec<String>,
}
+/// A way to stop the jobs of one [`Api`] from any thread, without waiting
+/// for its thread to be free: what a server that is shutting down calls
+/// before it waits for that thread. Cheap to clone.
+#[derive(Clone)]
+pub struct Stopper {
+ book: Arc<job::Book>,
+ cutouts: Arc<Cutouts>,
+}
+
+impl Stopper {
+ /// Asks every job this API started to stop at its next frame - its
+ /// own and no other, even where the export slot is shared with a
+ /// window. Each then ends with [`ErrorCode::Cancelled`]. How many
+ /// were running.
+ pub fn cancel_all(&self) -> usize {
+ let running = self.book.cancel_all();
+ if running > 0 {
+ // A job still finding masks is in the analysis, not the
+ // render; the analysis is this API's own.
+ self.cutouts.cancel();
+ }
+ running
+ }
+}
+
impl Api {
/// An API over this machine's app directories, reporting its jobs
/// through `events`.
@@ -176,7 +205,16 @@ impl Api {
/// An API over the given directories: a test's scratch, or an embedder
/// with a home of its own.
pub fn with_dirs(dirs: AppDirs, events: EventSink) -> Api {
+ let book = Arc::new(job::Book::default());
+ let events: EventSink = {
+ let book = Arc::clone(&book);
+ Arc::new(move |event: Event| {
+ book.record(&event);
+ events(event);
+ })
+ };
Api {
+ book,
titles: Titles::new(&dirs),
cutouts: Arc::new(Cutouts::new(&dirs.data)),
stabilizers: Arc::new(Stabilizers::new()),
@@ -207,6 +245,25 @@ impl Api {
}
}
+ /// What stops this API's jobs from another thread; see [`Stopper`].
+ pub fn stopper(&self) -> Stopper {
+ Stopper {
+ book: Arc::clone(&self.book),
+ cutouts: Arc::clone(&self.cutouts),
+ }
+ }
+
+ /// [`Request::ExportStatus`].
+ pub fn job_status(&mut self, job: &str) -> Result<JobStatus, ApiError> {
+ self.jobs.reap(&self.book);
+ self.book.status(job).ok_or_else(|| {
+ ApiError::new(
+ ErrorCode::NotFound,
+ format!("no job {job}: it never ran, or ended long ago"),
+ )
+ })
+ }
+
/// The export slot, for an embedder that shares it with a window.
pub fn exporter(&self) -> Exporter {
self.exporter.clone()
@@ -280,7 +337,7 @@ impl Api {
/// How many jobs are still running.
pub fn running(&mut self) -> usize {
- self.jobs.reap();
+ self.jobs.reap(&self.book);
self.jobs.threads.len()
}
@@ -293,7 +350,7 @@ impl Api {
/// Runs one request. The response is what the request is worth; a job
/// it begins reports through the sink from then on.
pub fn dispatch(&mut self, request: Request) -> Response {
- self.jobs.reap();
+ self.jobs.reap(&self.book);
self.run(request).into()
}
@@ -346,6 +403,7 @@ impl Api {
self.cancel(&job)?;
Ok(Reply::Done(Done {}))
}
+ Request::ExportStatus { job } => Ok(Reply::Job(self.job_status(&job)?)),
Request::PreviewFrame {
path,
time,
@@ -603,6 +661,10 @@ impl Api {
path: project_path,
output: spec.output.clone(),
};
+ self.book.start(
+ JobStatus::running(&started.job, &started.path, &started.output),
+ slot.cancel_handle(),
+ );
let job = ExportJob {
name: started.job.clone(),
path: started.path.clone(),
@@ -613,17 +675,22 @@ impl Api {
stabilizers: Arc::clone(&self.stabilizers),
events: Arc::clone(&self.events),
};
+ let book = Arc::clone(&self.book);
+ let name = started.job.clone();
let thread = std::thread::Builder::new()
.name(format!("export {}", started.job))
.spawn(move || job.run(slot))
- .map_err(|error| ApiError::failed(format!("could not start the export: {error}")))?;
+ .map_err(|error| {
+ book.abandon(&name);
+ ApiError::failed(format!("could not start the export: {error}"))
+ })?;
self.jobs.threads.push((started.job.clone(), thread));
Ok(started)
}
/// [`Request::ExportCancel`].
pub fn cancel(&mut self, job: &str) -> Result<(), ApiError> {
- self.jobs.reap();
+ self.jobs.reap(&self.book);
if !self.jobs.threads.iter().any(|(name, _)| name == job) {
return Err(ApiError::new(
ErrorCode::NotFound,
@@ -743,9 +810,16 @@ impl Jobs {
format!("j{}", self.minted)
}
- /// Forgets the threads that have ended.
- fn reap(&mut self) {
- self.threads.retain(|(_, thread)| !thread.is_finished());
+ /// Forgets the threads that have ended. One that ended without its
+ /// last event - a panic in the render - is booked as failed.
+ fn reap(&mut self, book: &job::Book) {
+ self.threads.retain(|(name, thread)| {
+ let ended = thread.is_finished();
+ if ended {
+ book.abandon(name);
+ }
+ !ended
+ });
}
/// Waits for the rest.
@@ -1429,7 +1503,24 @@ mod tests {
}
other => panic!("ended with {other:?}"),
}
+ drop(events);
assert_eq!(api.running(), 0);
+ // Asked afterwards, by a caller who was not listening, the job
+ // says the same.
+ let status = match ok(api.dispatch(Request::ExportStatus {
+ job: "j1".to_owned(),
+ })) {
+ Reply::Job(status) => status,
+ _ => panic!("not a job's status"),
+ };
+ assert_eq!(status.state, JobState::Failed);
+ assert_eq!(status.path, path);
+ assert_eq!(status.error.map(|error| error.code), Some(ErrorCode::Failed));
+ let unknown = err(api.dispatch(Request::ExportStatus {
+ job: "j9".to_owned(),
+ }));
+ assert_eq!(unknown.code, ErrorCode::NotFound);
+ assert_eq!(api.stopper().cancel_all(), 0, "nothing left to stop");
}
#[test]
diff --git a/src/crates/concat-api/src/message.rs b/src/crates/concat-api/src/message.rs
index f46332d2..0e0117b5 100644
--- a/src/crates/concat-api/src/message.rs
+++ b/src/crates/concat-api/src/message.rs
@@ -215,6 +215,14 @@ pub enum Request {
/// The job [`Started`] named.
job: String,
},
+ /// How a job stands: running and how far along, or how it ended. The
+ /// same news the events carry, for a caller that was not listening
+ /// when it came; the last few finished jobs are remembered.
+ #[serde(rename = "export.status")]
+ ExportStatus {
+ /// The job [`Started`] named.
+ job: String,
+ },
/// Composites the true frame at one instant. Written as a PNG to
/// `output` when there is one, and handed back inline as a
@@ -316,6 +324,8 @@ pub enum Reply {
Template(TemplateInfo),
/// [`Request::ExportRun`]: the job that is now running.
Started(Started),
+ /// [`Request::ExportStatus`].
+ Job(JobStatus),
/// [`Request::PreviewFrame`] with an output: the file written.
Written(Written),
/// [`Request::PreviewFrame`] without one: the picture itself.
@@ -387,6 +397,72 @@ pub struct Started {
pub output: String,
}
+/// Where a job is in its life.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum JobState {
+ /// Still working.
+ Running,
+ /// Wrote its file.
+ Done,
+ /// Stopped by [`Request::ExportCancel`].
+ Cancelled,
+ /// Ended without its file; `error` says why.
+ Failed,
+}
+
+/// How one job stands: [`Request::ExportStatus`]'s reply.
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct JobStatus {
+ /// The job's name.
+ pub job: String,
+ /// The project it works on.
+ pub path: String,
+ /// The file it writes.
+ pub output: String,
+ /// Running, or how it ended.
+ pub state: JobState,
+ /// What it is doing: "cutout" while masks are found, then "video",
+ /// "audio" and "mux". Absent until it has said.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub stage: Option<String>,
+ /// Frames rendered, in the video stage.
+ pub frame: i64,
+ /// Frames in all, in the video stage; 0 until known.
+ pub total: i64,
+ /// How far along the current stage is, `0..=1`; 1 once done.
+ pub fraction: f64,
+ /// The written file's picture size, once done.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub width: Option<u32>,
+ /// The written file's picture size, once done.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub height: Option<u32>,
+ /// Why it failed, or that it was cancelled.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub error: Option<ApiError>,
+}
+
+impl JobStatus {
+ /// A job that has just begun.
+ pub fn running(job: &str, path: &str, output: &str) -> JobStatus {
+ JobStatus {
+ job: job.to_owned(),
+ path: path.to_owned(),
+ output: output.to_owned(),
+ state: JobState::Running,
+ stage: None,
+ frame: 0,
+ total: 0,
+ fraction: 0.0,
+ width: None,
+ height: None,
+ error: None,
+ }
+ }
+}
+
/// A frame handed back without touching the disk.
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
diff --git a/src/crates/concat-server/src/grpc.rs b/src/crates/concat-server/src/grpc.rs
index 0e709f58..f1a976c9 100644
--- a/src/crates/concat-server/src/grpc.rs
+++ b/src/crates/concat-server/src/grpc.rs
@@ -29,6 +29,7 @@ use serde_json::{Value, json};
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Status};
+use crate::throttle::{Peer, Throttle};
use crate::{Hub, token};
/// The generated messages and service.
@@ -131,6 +132,7 @@ pub(crate) fn serve(
hub: Hub,
token: String,
stop: Arc<AtomicBool>,
+ throttle: Arc<Throttle>,
) -> Result<(SocketAddr, JoinHandle<()>), String> {
let could_not =
|error: &dyn std::fmt::Display| format!("could not listen on {address}: {error}");
@@ -158,7 +160,7 @@ pub(crate) fn serve(
return;
};
let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener);
- let service = ConcatServer::with_interceptor(Service { hub }, bearer(token));
+ let service = ConcatServer::with_interceptor(Service { hub }, bearer(token, throttle));
let stopped = async move {
while !stop.load(Ordering::SeqCst) {
tokio::time::sleep(Duration::from_millis(100)).await;
@@ -175,18 +177,33 @@ pub(crate) fn serve(
}
/// The check every call passes through: the token as a bearer, compared
-/// in constant time.
-fn bearer(token: String) -> impl Fn(Request<()>) -> Result<Request<()>, Status> + Clone {
+/// in constant time, from an address the throttle is not holding back.
+fn bearer(
+ token: String,
+ throttle: Arc<Throttle>,
+) -> impl Fn(Request<()>) -> Result<Request<()>, Status> + Clone {
let expected = format!("Bearer {token}");
move |request: Request<()>| {
+ let peer = request
+ .remote_addr()
+ .map(|address| Peer::Ip(address.ip()))
+ .unwrap_or(Peer::Local);
+ if let Some(wait) = throttle.wait(peer) {
+ return Err(Status::resource_exhausted(format!(
+ "too many wrong tokens from this address: try again in {} s",
+ wait.as_secs().max(1)
+ )));
+ }
let presented = request
.metadata()
.get("authorization")
.and_then(|value| value.to_str().ok())
.unwrap_or_default();
if token::matches(presented.as_bytes(), expected.as_bytes()) {
+ throttle.succeeded(peer);
Ok(request)
} else {
+ throttle.failed(peer);
Err(Status::unauthenticated("the token is missing or wrong"))
}
}
@@ -273,7 +290,7 @@ mod tests {
#[tokio::test]
async fn a_token_travels_as_a_bearer() {
- let (server, _scratch) = server(Some("open sesame"));
+ let (server, _scratch) = server(Some(crate::tests::TOKEN));
let address = server.grpc_addr().expect("listening");
let mut client = ConcatClient::connect(format!("http://{address}"))
.await
@@ -292,7 +309,7 @@ mod tests {
assert!(
client
- .call(with_bearer(version(), "open sesame"))
+ .call(with_bearer(version(), crate::tests::TOKEN))
.await
.is_ok()
);
diff --git a/src/crates/concat-server/src/hub.rs b/src/crates/concat-server/src/hub.rs
index 816671d6..2074a16a 100644
--- a/src/crates/concat-server/src/hub.rs
+++ b/src/crates/concat-server/src/hub.rs
@@ -19,7 +19,7 @@ use std::sync::mpsc::{self, Sender};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
-use concat_api::{Api, ApiError, ErrorCode, Event, EventSink, Request, Response};
+use concat_api::{Api, ApiError, ErrorCode, Event, EventSink, Request, Response, Stopper};
/// A way to hand an event to one caller. Returns false once the caller is
/// gone, after which it is dropped.
@@ -43,7 +43,7 @@ impl Hub {
/// fails, so does this.
pub(crate) fn start(
make: impl FnOnce(EventSink) -> Result<Api, String> + Send + 'static,
- ) -> Result<(Hub, JoinHandle<()>), String> {
+ ) -> Result<(Hub, JoinHandle<()>, Stopper), String> {
let (tasks, queue) = mpsc::channel::<Task>();
let subscribers: Arc<Mutex<Vec<Subscriber>>> = Arc::new(Mutex::new(Vec::new()));
let sink: EventSink = {
@@ -66,13 +66,13 @@ impl Hub {
}
})
};
- let (made, ready) = mpsc::channel::<Result<(), String>>();
+ let (made, ready) = mpsc::channel::<Result<Stopper, String>>();
let thread = std::thread::Builder::new()
.name("concat api".to_owned())
.spawn(move || {
let mut api = match make(sink) {
Ok(api) => {
- let _ = made.send(Ok(()));
+ let _ = made.send(Ok(api.stopper()));
api
}
Err(error) => {
@@ -107,7 +107,7 @@ impl Hub {
api.finish();
})
.map_err(|error| format!("could not start the API's thread: {error}"))?;
- ready
+ let stopper = ready
.recv()
.unwrap_or_else(|_| Err("the API's thread ended before it began".to_owned()))?;
Ok((
@@ -116,6 +116,7 @@ impl Hub {
subscribers,
},
thread,
+ stopper,
))
}
@@ -151,6 +152,14 @@ impl Hub {
}
}
+ /// Forgets a subscriber at once, for a caller that has hung up: the
+ /// fan-out would only notice at the next event, which may be never.
+ pub fn unsubscribe(&self, subscriber: &Subscriber) {
+ if let Ok(mut subscribers) = self.subscribers.lock() {
+ subscribers.retain(|kept| !Arc::ptr_eq(kept, subscriber));
+ }
+ }
+
/// How many callers are receiving events.
pub fn subscribers(&self) -> usize {
self.subscribers.lock().map(|list| list.len()).unwrap_or(0)
diff --git a/src/crates/concat-server/src/json.rs b/src/crates/concat-server/src/json.rs
index 8955a752..eb90746d 100644
--- a/src/crates/concat-server/src/json.rs
+++ b/src/crates/concat-server/src/json.rs
@@ -4,249 +4,369 @@
//! JSON-RPC lines on a socket: the stdin transport, for several callers at
//! once.
//!
-//! A connection is two threads: one reads lines and answers each through
-//! the [`Hub`], one writes whatever is put in the connection's outbox -
-//! responses and events alike - so a job's event from another thread never
-//! interleaves with a response half-written. The wire is exactly what
-//! `concat-cli api` reads and writes; see `concat_api::rpc`.
-//!
//! A connection's first line is `{"jsonrpc": "2.0", "id": 1, "method":
//! "auth", "params": {"token": "..."}}`, with the server's token. Anything
//! else first, or the wrong token, is answered with an `unauthorized` error
//! and the connection is closed. `auth` is the transport's word, not the
-//! API's: it never reaches the hub.
-
-use std::io::{BufRead, BufReader, Write};
-use std::net::TcpListener;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+//! API's: it never reaches the hub. The wire after it is exactly what
+//! `concat-cli api` reads and writes; see `concat_api::rpc`.
+//!
+//! A caller that has not presented its token yet holds as little as it
+//! can: one thread and the socket it arrived on, for at most
+//! [`AUTH_TIMEOUT`] all told - however slowly it dribbles its line in -
+//! and at most [`MAX_PENDING`] of them wait at once. Only a caller that is
+//! in takes one of the [`MAX_CONNECTIONS`] seats, gets the handles a stop
+//! needs to reach it, and is subscribed to events; all three are given
+//! back the moment it hangs up, not when the server stops. An address
+//! that keeps presenting wrong tokens waits longer each time (see
+//! [`crate::throttle`]). The accept loop backs off when accepting fails -
+//! out of file descriptors, say - instead of spinning on the error.
+//!
+//! Once in, a connection is two threads: one reads lines and answers each
+//! through the [`Hub`], one writes whatever is put in the connection's
+//! outbox - responses and events alike - so a job's event from another
+//! thread never interleaves with a response half-written. The outbox is
+//! bounded and a write gives up after [`WRITE_TIMEOUT`], so a caller that
+//! stops reading is hung up on rather than holding anything up.
+
+use std::io::{BufRead, BufReader, Read, Write};
+use std::net::{Shutdown, TcpListener, TcpStream};
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::mpsc::{self, SyncSender};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
-use std::time::Duration;
+use std::time::{Duration, Instant};
use concat_api::rpc::{Call, Id, Message};
use concat_api::{ApiError, Done, ErrorCode, Reply, Response};
use serde_json::Value;
-use crate::{Connections, Hub, token};
+use crate::throttle::{Peer, Throttle};
+use crate::{Connections, Hub, Subscriber, token};
/// The longest line a caller may send once it is in: a document-sized
/// edit fits, a buffer grown to whatever arrives does not.
pub const MAX_LINE: usize = 4 * 1024 * 1024;
/// The longest first line: the `auth` call is a few dozen bytes.
pub const MAX_AUTH_LINE: usize = 4 * 1024;
-/// How long a caller has to present its token before the connection is
-/// closed, so one that connects and says nothing holds no thread.
-pub const AUTH_TIMEOUT: Duration = Duration::from_secs(10);
-/// How many callers may be connected at once; the next is refused with
-/// `busy` and closed.
+/// How long a caller has, in all, to present its token before the
+/// connection is closed.
+pub const AUTH_TIMEOUT: Duration = Duration::from_secs(5);
+/// How many callers may be in at once; the next to present its token is
+/// refused with `busy` and closed.
pub const MAX_CONNECTIONS: usize = 64;
+/// How many connections may be waiting to present their token at once;
+/// one more is refused with `busy` as it arrives.
+pub const MAX_PENDING: usize = 32;
+/// How long one write to a caller may take before the caller is taken to
+/// have stopped reading and is hung up on.
+pub const WRITE_TIMEOUT: Duration = Duration::from_secs(10);
/// How many lines may wait for a caller that is not reading them before
/// the connection is closed rather than the queue grown.
const OUTBOX_LINES: usize = 256;
+/// The accept loop's first and longest pause after a failed accept.
+const BACKOFF_FIRST: Duration = Duration::from_millis(10);
+const BACKOFF_LONGEST: Duration = Duration::from_secs(1);
+
+/// What the JSON transport needs of a stream: TCP and a Unix socket alike.
+pub(crate) trait Socket: Read + Write + Send + Sized + 'static {
+ fn duplicate(&self) -> std::io::Result<Self>;
+ fn hang_up(&self);
+ fn read_timeout(&self, timeout: Option<Duration>) -> std::io::Result<()>;
+ fn write_timeout(&self, timeout: Option<Duration>) -> std::io::Result<()>;
+ fn peer(&self) -> Peer;
+}
-/// One of the [`MAX_CONNECTIONS`] seats; given back when dropped.
-struct Seat(Arc<AtomicUsize>);
+impl Socket for TcpStream {
+ fn duplicate(&self) -> std::io::Result<Self> {
+ self.try_clone()
+ }
+ fn hang_up(&self) {
+ let _ = self.shutdown(Shutdown::Both);
+ }
+ fn read_timeout(&self, timeout: Option<Duration>) -> std::io::Result<()> {
+ self.set_read_timeout(timeout)
+ }
+ fn write_timeout(&self, timeout: Option<Duration>) -> std::io::Result<()> {
+ self.set_write_timeout(timeout)
+ }
+ fn peer(&self) -> Peer {
+ self.peer_addr()
+ .map(|address| Peer::Ip(address.ip()))
+ .unwrap_or(Peer::Local)
+ }
+}
-impl Seat {
- fn take(seats: &Arc<AtomicUsize>) -> Option<Seat> {
- let mut taken = seats.load(Ordering::SeqCst);
+#[cfg(unix)]
+impl Socket for std::os::unix::net::UnixStream {
+ fn duplicate(&self) -> std::io::Result<Self> {
+ self.try_clone()
+ }
+ fn hang_up(&self) {
+ let _ = self.shutdown(Shutdown::Both);
+ }
+ fn read_timeout(&self, timeout: Option<Duration>) -> std::io::Result<()> {
+ self.set_read_timeout(timeout)
+ }
+ fn write_timeout(&self, timeout: Option<Duration>) -> std::io::Result<()> {
+ self.set_write_timeout(timeout)
+ }
+ fn peer(&self) -> Peer {
+ Peer::Local
+ }
+}
+
+/// What every connection of one server shares.
+#[derive(Clone)]
+pub(crate) struct Door {
+ pub(crate) hub: Hub,
+ pub(crate) token: String,
+ pub(crate) stop: Arc<AtomicBool>,
+ pub(crate) connections: Connections,
+ pub(crate) seats: Arc<AtomicUsize>,
+ pub(crate) pending: Arc<AtomicUsize>,
+ pub(crate) throttle: Arc<Throttle>,
+ pub(crate) next_id: Arc<AtomicU64>,
+}
+
+/// One of a counted number of places, given back when dropped.
+struct Place(Arc<AtomicUsize>);
+
+impl Place {
+ fn take(count: &Arc<AtomicUsize>, cap: usize) -> Option<Place> {
+ let mut taken = count.load(Ordering::SeqCst);
loop {
- if taken >= MAX_CONNECTIONS {
+ if taken >= cap {
return None;
}
- match seats.compare_exchange(taken, taken + 1, Ordering::SeqCst, Ordering::SeqCst) {
- Ok(_) => return Some(Seat(Arc::clone(seats))),
+ match count.compare_exchange(taken, taken + 1, Ordering::SeqCst, Ordering::SeqCst) {
+ Ok(_) => return Some(Place(Arc::clone(count))),
Err(now) => taken = now,
}
}
}
}
-impl Drop for Seat {
+impl Drop for Place {
fn drop(&mut self) {
self.0.fetch_sub(1, Ordering::SeqCst);
}
}
-/// Tells a caller there is no seat, and hangs up.
-fn refuse_full(mut writer: impl Write) {
+/// A connection's entry in the server's registry of ways to close it,
+/// taken out when the connection ends.
+struct Registered {
+ connections: Connections,
+ id: u64,
+}
+
+impl Drop for Registered {
+ fn drop(&mut self) {
+ if let Ok(mut connections) = self.connections.lock() {
+ connections.remove(&self.id);
+ }
+ }
+}
+
+/// Writes one refusal and lets the socket go. Best effort: the line is
+/// small enough for the socket's buffer, and the write may not wait.
+fn refuse(mut stream: impl Socket, id: Option<Id>, code: ErrorCode, why: String) {
+ let _ = stream.write_timeout(Some(Duration::from_millis(500)));
let line = Message::Reply {
- id: None,
- response: Response::Error(ApiError::new(
- ErrorCode::Busy,
- format!("{MAX_CONNECTIONS} callers are connected already"),
- )),
+ id,
+ response: Response::Error(ApiError::new(code, why)),
}
.to_json();
- let _ = writeln!(writer, "{line}");
- let _ = writer.flush();
+ let _ = writeln!(stream, "{line}");
+ let _ = stream.flush();
+ stream.hang_up();
}
/// Accepts JSON-RPC connections on `listener` until `stop` is set.
-pub(crate) fn serve_tcp(
- listener: TcpListener,
- hub: Hub,
- token: String,
- stop: Arc<AtomicBool>,
- connections: Connections,
- seats: Arc<AtomicUsize>,
-) -> JoinHandle<()> {
- std::thread::spawn(move || {
- for stream in listener.incoming() {
- if stop.load(Ordering::SeqCst) {
- break;
- }
- let Ok(stream) = stream else { continue };
- let _ = stream.set_nodelay(true);
- let Some(seat) = Seat::take(&seats) else {
- refuse_full(stream);
- continue;
- };
- let Ok(reader) = stream.try_clone() else {
- continue;
- };
- let Ok(closer) = stream.try_clone() else {
- continue;
- };
- let Ok(ender) = stream.try_clone() else {
- continue;
- };
- let Ok(timer) = stream.try_clone() else {
- continue;
- };
- if let Ok(mut connections) = connections.lock() {
- connections.push(Box::new(move || {
- let _ = closer.shutdown(std::net::Shutdown::Both);
- }));
- }
- // The handshake has this long; once it is done the caller may
- // be silent for as long as it likes.
- let _ = stream.set_read_timeout(Some(AUTH_TIMEOUT));
- let hub = hub.clone();
- let token = token.clone();
- std::thread::spawn(move || {
- let _seat = seat;
- connection(BufReader::new(reader), stream, hub, token, move || {
- let _ = timer.set_read_timeout(None);
- });
- // The registry above still holds a handle, so the socket is
- // shut here rather than merely dropped: the caller sees EOF.
- let _ = ender.shutdown(std::net::Shutdown::Both);
- });
- }
- })
+pub(crate) fn serve_tcp(listener: TcpListener, door: Door) -> std::io::Result<JoinHandle<()>> {
+ std::thread::Builder::new()
+ .name("concat json accept".to_owned())
+ .spawn(move || accept_loop(|| listener.accept().map(|(stream, _)| stream), door))
}
/// Accepts JSON-RPC connections on a Unix socket until `stop` is set.
#[cfg(unix)]
pub(crate) fn serve_unix(
listener: std::os::unix::net::UnixListener,
- hub: Hub,
- token: String,
- stop: Arc<AtomicBool>,
- connections: Connections,
- seats: Arc<AtomicUsize>,
-) -> JoinHandle<()> {
- std::thread::spawn(move || {
- for stream in listener.incoming() {
- if stop.load(Ordering::SeqCst) {
- break;
+ door: Door,
+) -> std::io::Result<JoinHandle<()>> {
+ std::thread::Builder::new()
+ .name("concat socket accept".to_owned())
+ .spawn(move || accept_loop(|| listener.accept().map(|(stream, _)| stream), door))
+}
+
+/// Takes connections until the server stops. A failed accept - the
+/// process out of descriptors, the peer gone before it was taken - is
+/// waited out, longer each time it repeats, so the loop never spins.
+fn accept_loop<S: Socket>(mut accept: impl FnMut() -> std::io::Result<S>, door: Door) {
+ let mut backoff = Duration::ZERO;
+ loop {
+ let accepted = accept();
+ if door.stop.load(Ordering::SeqCst) {
+ break;
+ }
+ match accepted {
+ Ok(stream) => {
+ backoff = Duration::ZERO;
+ admit(stream, &door);
}
- let Ok(stream) = stream else { continue };
- let Some(seat) = Seat::take(&seats) else {
- refuse_full(stream);
- continue;
- };
- let Ok(reader) = stream.try_clone() else {
- continue;
- };
- let Ok(closer) = stream.try_clone() else {
- continue;
- };
- let Ok(ender) = stream.try_clone() else {
- continue;
- };
- let Ok(timer) = stream.try_clone() else {
- continue;
- };
- if let Ok(mut connections) = connections.lock() {
- connections.push(Box::new(move || {
- let _ = closer.shutdown(std::net::Shutdown::Both);
- }));
+ Err(_) => {
+ backoff = (backoff * 2).clamp(BACKOFF_FIRST, BACKOFF_LONGEST);
+ std::thread::sleep(backoff);
}
- // The handshake has this long; once it is done the caller may
- // be silent for as long as it likes.
- let _ = stream.set_read_timeout(Some(AUTH_TIMEOUT));
- let hub = hub.clone();
- let token = token.clone();
- std::thread::spawn(move || {
- let _seat = seat;
- connection(BufReader::new(reader), stream, hub, token, move || {
- let _ = timer.set_read_timeout(None);
- });
- // The registry above still holds a handle, so the socket is
- // shut here rather than merely dropped: the caller sees EOF.
- let _ = ender.shutdown(std::net::Shutdown::Both);
- });
}
- })
+ }
+}
+
+/// Lets one connection wait for its token, if it may.
+fn admit<S: Socket>(stream: S, door: &Door) {
+ let peer = stream.peer();
+ if let Some(wait) = door.throttle.wait(peer) {
+ refuse(
+ stream,
+ None,
+ ErrorCode::Unauthorized,
+ format!(
+ "too many wrong tokens from this address: try again in {} s",
+ wait.as_secs().max(1)
+ ),
+ );
+ return;
+ }
+ let Some(pending) = Place::take(&door.pending, MAX_PENDING) else {
+ refuse(
+ stream,
+ None,
+ ErrorCode::Busy,
+ format!("{MAX_PENDING} connections are waiting to present a token already"),
+ );
+ return;
+ };
+ let id = door.next_id.fetch_add(1, Ordering::SeqCst);
+ let door = door.clone();
+ // A thread that could not be started drops the stream with the
+ // closure, which closes it.
+ let _ = std::thread::Builder::new()
+ .name(format!("concat json {id}"))
+ .spawn(move || connection(stream, door, id, peer, pending));
}
/// One caller, start to end: the handshake, then a call per line.
-/// `authenticated` runs once the token has been presented, for the
-/// transport to lift the handshake's timeout.
-fn connection(
- mut reader: impl BufRead,
- writer: impl Write + Send + 'static,
- hub: Hub,
- token: String,
- authenticated: impl FnOnce(),
-) {
- let outbox = Outbox::start(writer);
-
- let first = loop {
- match read_line(&mut reader, MAX_AUTH_LINE) {
- None => {
- outbox.close();
- return;
+fn connection<S: Socket>(mut stream: S, door: Door, id: u64, peer: Peer, pending: Place) {
+ let deadline = Instant::now() + AUTH_TIMEOUT;
+ let Some((first, rest)) = first_line(&mut stream, deadline) else {
+ // Silent, too slow, too long, or gone: the socket closes as it
+ // drops, and nothing else was ever held for it.
+ return;
+ };
+ let (call, outcome) = handshake(&first, &door.token);
+ if let Err(error) = outcome {
+ door.throttle.failed(peer);
+ refuse(stream, call, error.code, error.message);
+ return;
+ }
+ door.throttle.succeeded(peer);
+ drop(pending);
+ let Some(_seat) = Place::take(&door.seats, MAX_CONNECTIONS) else {
+ refuse(
+ stream,
+ call,
+ ErrorCode::Busy,
+ format!("{MAX_CONNECTIONS} callers are connected already"),
+ );
+ return;
+ };
+ if door.stop.load(Ordering::SeqCst) {
+ return;
+ }
+ // In: the handles the rest needs. A stop reaches the reader through
+ // the registry; the writer has its own handle, and its own timeout.
+ let (Ok(writer), Ok(closer)) = (stream.duplicate(), stream.duplicate()) else {
+ return;
+ };
+ let _registered = match door.connections.lock() {
+ Ok(mut connections) => {
+ connections.insert(id, Box::new(move || closer.hang_up()));
+ Registered {
+ connections: Arc::clone(&door.connections),
+ id,
}
- Some(line) if line.trim().is_empty() => continue,
- Some(line) => break line,
}
+ Err(_) => return,
};
- let (id, outcome) = handshake(&first, &token);
- let ok = outcome.is_ok();
- outbox.send(Message::Reply {
- id,
- response: outcome.into(),
- });
- if !ok {
- outbox.close();
+ if stream.read_timeout(None).is_err() || writer.write_timeout(Some(WRITE_TIMEOUT)).is_err() {
return;
}
- authenticated();
+ let Some(outbox) = Outbox::start(writer, id) else {
+ return;
+ };
+ outbox.send(Message::Reply {
+ id: call,
+ response: Response::Result(Reply::Done(Done {})),
+ });
// Events reach this caller from here on: the outbox is shared with the
// hub's fan-out, which drops it once a send fails.
let events = outbox.clone();
- hub.subscribe(Arc::new(move |event| {
- events.send(Message::Event(event.clone()))
- }));
+ let subscriber: Subscriber = Arc::new(move |event| events.send(Message::Event(event.clone())));
+ door.hub.subscribe(Arc::clone(&subscriber));
+ let mut reader = BufReader::new(std::io::Cursor::new(rest).chain(stream));
while let Some(line) = read_line(&mut reader, MAX_LINE) {
if line.trim().is_empty() {
continue;
}
let (id, response) = match Call::parse(&line) {
- Ok(call) => (call.id, hub.call(call.request)),
+ Ok(call) => (call.id, door.hub.call(call.request)),
Err((id, error)) => (id, Response::Error(error)),
};
if !outbox.send(Message::Reply { id, response }) {
break;
}
}
+ door.hub.unsubscribe(&subscriber);
outbox.close();
+ // The caller sees EOF even where its own end is still open.
+ reader.into_inner().into_inner().1.hang_up();
+}
+
+/// The first line that is not blank, read before `deadline` - a deadline
+/// for the whole line, not for each read, so a caller cannot keep a
+/// handshake open by sending a byte at a time. The bytes that arrived
+/// after the line come back with it, for the reader that follows.
+fn first_line<S: Socket>(stream: &mut S, deadline: Instant) -> Option<(String, Vec<u8>)> {
+ let mut buffer: Vec<u8> = Vec::new();
+ let mut chunk = [0u8; 512];
+ loop {
+ while let Some(end) = buffer.iter().position(|&byte| byte == b'\n') {
+ let rest = buffer.split_off(end + 1);
+ let line = String::from_utf8(std::mem::take(&mut buffer)).ok()?;
+ let line = line.trim_end_matches(['\r', '\n']);
+ if !line.trim().is_empty() {
+ return Some((line.to_owned(), rest));
+ }
+ buffer = rest;
+ }
+ if buffer.len() > MAX_AUTH_LINE {
+ return None;
+ }
+ let left = deadline.checked_duration_since(Instant::now())?;
+ if left.is_zero() {
+ return None;
+ }
+ stream.read_timeout(Some(left)).ok()?;
+ match stream.read(&mut chunk) {
+ Ok(0) => return None,
+ Ok(read) => buffer.extend_from_slice(&chunk[..read]),
+ Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
+ Err(_) => return None,
+ }
+ }
}
/// One line of at most `cap` bytes, its newline taken off. `None` at the
@@ -264,7 +384,7 @@ fn read_line(reader: &mut impl BufRead, cap: usize) -> Option<String> {
/// Reads the first line as the `auth` call and checks its token, in
/// constant time.
-fn handshake(line: &str, token: &str) -> (Option<Id>, Result<Reply, ApiError>) {
+fn handshake(line: &str, token: &str) -> (Option<Id>, Result<(), ApiError>) {
let refuse = |id: Option<Id>, why: &str| {
(
id,
@@ -284,7 +404,7 @@ fn handshake(line: &str, token: &str) -> (Option<Id>, Result<Reply, ApiError>) {
.or_else(|| object.get("token"))
.and_then(Value::as_str);
if token::matches(presented.unwrap_or_default().as_bytes(), token.as_bytes()) {
- (id, Ok(Reply::Done(Done {})))
+ (id, Ok(()))
} else {
refuse(id, "wrong token")
}
@@ -298,21 +418,31 @@ struct Outbox {
}
impl Outbox {
- fn start(mut writer: impl Write + Send + 'static) -> Outbox {
+ /// `None` when no thread could be started for it.
+ fn start<S: Socket>(mut writer: S, id: u64) -> Option<Outbox> {
// Bounded: a caller that stops reading is hung up on at the cap,
// not kept in memory line by line.
let (queue, messages) = mpsc::sync_channel::<Message>(OUTBOX_LINES);
- let thread = std::thread::spawn(move || {
- for message in messages {
- if writeln!(writer, "{}", message.to_json()).is_err() || writer.flush().is_err() {
- break;
+ let thread = std::thread::Builder::new()
+ .name(format!("concat json {id} out"))
+ .spawn(move || {
+ for message in messages {
+ if writeln!(writer, "{}", message.to_json()).is_err()
+ || writer.flush().is_err()
+ {
+ // Gone, or stalled past the write timeout: hang up
+ // so the reader ends too, and drop the queue so
+ // every later send says the caller is gone.
+ writer.hang_up();
+ break;
+ }
}
- }
- });
- Outbox {
+ })
+ .ok()?;
+ Some(Outbox {
queue: Arc::new(Mutex::new(Some(queue))),
writer: Arc::new(Mutex::new(Some(thread))),
- }
+ })
}
/// Queues one line. False once the caller is gone.
@@ -326,8 +456,8 @@ impl Outbox {
}
/// Ends the queue and waits for the writer to finish what is left, so
- /// the last reply is on the wire when this returns. The fan-out forgets
- /// this caller at its next event.
+ /// the last reply is on the wire when this returns - or the write
+ /// timeout has given up on it.
fn close(&self) {
if let Ok(mut queue) = self.queue.lock() {
queue.take();
@@ -342,8 +472,8 @@ impl Outbox {
#[cfg(test)]
mod tests {
use super::*;
- use crate::tests::server;
- use std::net::TcpStream;
+ use crate::tests::{TOKEN, server};
+ use crate::throttle::FREE_FAILURES;
/// A client on `server`: a line in, a line out.
struct Client {
@@ -355,10 +485,7 @@ mod tests {
/// A connection with the server's token presented, ready to call.
fn connect(server: &crate::Server) -> Client {
let mut client = Client::reach(server);
- let welcomed = client.ask(&format!(
- r#"{{"jsonrpc":"2.0","id":0,"method":"auth","params":{{"token":"{}"}}}}"#,
- server.token()
- ));
+ let welcomed = client.auth(server.token());
assert_eq!(welcomed["result"], serde_json::json!({}), "{welcomed}");
client
}
@@ -367,18 +494,52 @@ mod tests {
fn reach(server: &crate::Server) -> Client {
let stream =
TcpStream::connect(server.json_addr().expect("listening")).expect("connects");
+ stream
+ .set_read_timeout(Some(Duration::from_secs(30)))
+ .expect("a timeout");
Client {
reader: BufReader::new(stream.try_clone().expect("clone")),
writer: stream,
}
}
+ fn auth(&mut self, token: &str) -> Value {
+ self.ask(
+ &serde_json::json!({
+ "jsonrpc": "2.0", "id": 0, "method": "auth", "params": { "token": token },
+ })
+ .to_string(),
+ )
+ }
+
fn ask(&mut self, line: &str) -> Value {
writeln!(self.writer, "{line}").expect("writes");
+ self.read()
+ }
+
+ fn read(&mut self) -> Value {
let mut reply = String::new();
self.reader.read_line(&mut reply).expect("reads");
- serde_json::from_str(&reply).expect("a JSON line")
+ serde_json::from_str(&reply).unwrap_or_else(|_| panic!("a JSON line: {reply:?}"))
}
+
+ /// Whether the server has hung up: EOF, or a reset.
+ fn closed(&mut self) -> bool {
+ let mut after = String::new();
+ matches!(self.reader.read_line(&mut after), Ok(0) | Err(_))
+ }
+ }
+
+ /// Waits up to five seconds for `condition`.
+ fn eventually(condition: impl Fn() -> bool) -> bool {
+ let until = Instant::now() + Duration::from_secs(5);
+ while Instant::now() < until {
+ if condition() {
+ return true;
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ condition()
}
#[test]
@@ -389,12 +550,7 @@ mod tests {
// The write may fail part way once the server hangs up; that is
// the point.
let _ = writeln!(client.writer, "{long}");
- let mut after = String::new();
- assert_eq!(
- client.reader.read_line(&mut after).unwrap_or(0),
- 0,
- "closed without an answer"
- );
+ assert!(client.closed(), "closed without an answer");
server.stop();
}
@@ -404,12 +560,107 @@ mod tests {
let seated: Vec<Client> = (0..MAX_CONNECTIONS)
.map(|_| Client::connect(&server))
.collect();
+ assert_eq!(server.connections(), MAX_CONNECTIONS);
let mut late = Client::reach(&server);
- let mut line = String::new();
- late.reader.read_line(&mut line).expect("a refusal");
- let refused: Value = serde_json::from_str(&line).expect("a JSON line");
+ let refused = late.auth(server.token());
assert_eq!(refused["error"]["data"]["code"], "busy", "{refused}");
+ assert!(late.closed());
drop(seated);
+ // A seat given back is taken again.
+ assert!(eventually(|| server.connections() == 0), "the seats come back");
+ let _back = Client::connect(&server);
+ server.stop();
+ }
+
+ #[test]
+ fn a_silent_connection_is_dropped_at_the_deadline_and_holds_no_seat() {
+ let (server, _scratch) = server(None);
+ let started = Instant::now();
+ let mut silent = Client::reach(&server);
+ // Waiting callers are not seated callers.
+ let mut talker = Client::connect(&server);
+ assert_eq!(server.connections(), 1);
+ assert!(silent.closed(), "hung up on");
+ let waited = started.elapsed();
+ assert!(
+ waited >= AUTH_TIMEOUT - Duration::from_millis(100)
+ && waited < AUTH_TIMEOUT + Duration::from_secs(3),
+ "after {waited:?}"
+ );
+ let reply = talker.ask(r#"{"jsonrpc":"2.0","id":1,"method":"version"}"#);
+ assert_eq!(reply["id"], 1);
+ server.stop();
+ }
+
+ #[test]
+ fn a_dribbled_token_does_not_stretch_the_deadline() {
+ let (server, _scratch) = server(None);
+ let mut slow = Client::reach(&server);
+ let started = Instant::now();
+ // A byte every half second resets a per-read timeout forever; the
+ // deadline is for the whole line.
+ let mut hung_up = false;
+ while started.elapsed() < AUTH_TIMEOUT + Duration::from_secs(3) {
+ if slow.writer.write_all(b" ").is_err() {
+ hung_up = true;
+ break;
+ }
+ std::thread::sleep(Duration::from_millis(500));
+ }
+ assert!(hung_up || slow.closed(), "still open after the deadline");
+ assert!(started.elapsed() < AUTH_TIMEOUT + Duration::from_secs(3));
+ server.stop();
+ }
+
+ #[test]
+ fn waiting_connections_are_capped_and_a_closed_one_leaves_nothing_behind() {
+ let (server, _scratch) = server(None);
+ let waiting: Vec<Client> = (0..MAX_PENDING).map(|_| Client::reach(&server)).collect();
+ assert!(eventually(|| server.pending() == MAX_PENDING));
+ let mut over = Client::reach(&server);
+ let refused = over.read();
+ assert_eq!(refused["error"]["data"]["code"], "busy", "{refused}");
+ drop(waiting);
+ // Each waiting thread sees its caller go and gives its place back.
+ assert!(eventually(|| server.pending() == 0), "{}", server.pending());
+ // In and out again: nothing stays registered for a stop to find.
+ for _ in 0..8 {
+ let client = Client::connect(&server);
+ drop(client);
+ }
+ assert!(eventually(|| server.registered() == 0 && server.connections() == 0));
+ assert_eq!(server.hub().subscribers(), 0, "nobody left subscribed");
+ server.stop();
+ }
+
+ #[test]
+ fn wrong_tokens_are_throttled_and_the_right_one_clears_it() {
+ let (server, _scratch) = server(None);
+ for _ in 0..=FREE_FAILURES {
+ let mut wrong = Client::reach(&server);
+ let refused = wrong.auth("not the token at all, no");
+ assert_eq!(refused["error"]["data"]["code"], "unauthorized");
+ assert_eq!(refused["error"]["message"], "wrong token");
+ }
+ // Now even the right token waits: the refusal comes as the
+ // connection is taken, before a token is read.
+ let mut early = Client::reach(&server);
+ let refused = early.read();
+ assert_eq!(refused["error"]["data"]["code"], "unauthorized");
+ assert!(
+ refused["error"]["message"]
+ .as_str()
+ .is_some_and(|message| message.contains("try again")),
+ "{refused}"
+ );
+ std::thread::sleep(crate::throttle::FIRST_WAIT + Duration::from_millis(100));
+ let mut right = Client::connect(&server);
+ let reply = right.ask(r#"{"jsonrpc":"2.0","id":2,"method":"version"}"#);
+ assert_eq!(reply["id"], 2);
+ // A clean slate: one more wrong token is forgiven again.
+ let mut slip = Client::reach(&server);
+ assert_eq!(slip.auth("still not it, sorry")["error"]["message"], "wrong token");
+ let _ = Client::connect(&server);
server.stop();
}
@@ -431,6 +682,23 @@ mod tests {
server.stop();
}
+ #[test]
+ fn the_auth_line_and_the_first_call_may_arrive_together() {
+ let (server, _scratch) = server(None);
+ let mut client = Client::reach(&server);
+ write!(
+ client.writer,
+ "\n{}\n{{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"version\"}}\n",
+ serde_json::json!({
+ "jsonrpc": "2.0", "id": 0, "method": "auth", "params": { "token": server.token() },
+ }),
+ )
+ .expect("writes");
+ assert_eq!(client.read()["result"], serde_json::json!({}));
+ assert_eq!(client.read()["id"], 1);
+ server.stop();
+ }
+
#[test]
fn two_callers_share_one_api() {
let (server, scratch) = server(None);
@@ -466,27 +734,20 @@ mod tests {
#[test]
fn a_token_is_presented_first_or_the_connection_ends() {
- let (server, _scratch) = server(Some("open sesame"));
+ let (server, _scratch) = server(Some(TOKEN));
let mut silent = Client::reach(&server);
let refused = silent.ask(r#"{"jsonrpc":"2.0","id":1,"method":"version"}"#);
assert_eq!(refused["error"]["data"]["code"], "unauthorized");
- let mut after = String::new();
- assert_eq!(
- silent.reader.read_line(&mut after).expect("eof"),
- 0,
- "closed"
- );
+ assert_eq!(refused["id"], 1, "the refusal names the call");
+ assert!(silent.closed());
let mut wrong = Client::reach(&server);
- let refused =
- wrong.ask(r#"{"jsonrpc":"2.0","id":1,"method":"auth","params":{"token":"open"}}"#);
+ let refused = wrong.auth(&TOKEN[1..]);
assert_eq!(refused["error"]["data"]["code"], "unauthorized");
- assert_eq!(wrong.reader.read_line(&mut after).expect("eof"), 0);
+ assert!(wrong.closed());
let mut right = Client::reach(&server);
- let welcomed = right
- .ask(r#"{"jsonrpc":"2.0","id":1,"method":"auth","params":{"token":"open sesame"}}"#);
- assert_eq!(welcomed["result"], serde_json::json!({}));
+ assert_eq!(right.auth(TOKEN)["result"], serde_json::json!({}));
let reply = right.ask(r#"{"jsonrpc":"2.0","id":2,"method":"version"}"#);
assert_eq!(reply["id"], 2);
assert_eq!(server.connections(), 1);
@@ -505,4 +766,12 @@ mod tests {
assert_eq!(reply["result"]["apiVersion"], concat_api::API_VERSION);
server.stop();
}
+
+ #[test]
+ fn a_stop_reaches_connected_callers() {
+ let (server, _scratch) = server(None);
+ let mut client = Client::connect(&server);
+ server.stop();
+ assert!(client.closed());
+ }
}
diff --git a/src/crates/concat-server/src/lib.rs b/src/crates/concat-server/src/lib.rs
index a1c02c3b..06d91b74 100644
--- a/src/crates/concat-server/src/lib.rs
+++ b/src/crates/concat-server/src/lib.rs
@@ -26,32 +26,76 @@
//! easily as this one does. A server given no token mints one, 128 bits
//! from the operating system's randomness, that only the process that
//! started it knows; [`Server::token`] is how that process passes it on,
-//! to a page or a terminal. The token is compared in constant time,
-//! whichever transport carries it. The API writes only under the roots
-//! the server is given - the user's home unless told otherwise
-//! ([`Config::roots`]) - and a frame or an export is bounded in size.
-//! The JSON transport reads a line of at most [`json::MAX_LINE`] bytes,
-//! gives a caller [`json::AUTH_TIMEOUT`] to present its token, seats at
-//! most [`json::MAX_CONNECTIONS`] callers at once and hangs up on one
-//! that stops reading its replies. There is no encryption: a bind off
-//! loopback belongs behind something that provides it.
+//! to a page or a terminal. A token that is given must pass
+//! [`check_token`]: twenty characters at least, and not a repeated
+//! pattern. It is compared in constant time, whichever transport carries
+//! it, and an address that keeps presenting wrong ones waits longer each
+//! time ([`throttle`]). The API writes only under the roots the server is
+//! given - the user's home unless told otherwise ([`Config::roots`]) - and
+//! a frame or an export is bounded in size. The JSON transport reads a
+//! line of at most [`json::MAX_LINE`] bytes, gives a caller
+//! [`json::AUTH_TIMEOUT`] to present its token, holds nothing for a caller
+//! that has not, seats at most [`json::MAX_CONNECTIONS`] callers at once
+//! and hangs up on one that stops reading its replies. There is no
+//! encryption: a bind off loopback belongs behind something that provides
+//! it, and [`is_exposed`] is how a front end says so.
mod hub;
pub mod json;
+pub mod throttle;
mod token;
#[cfg(feature = "grpc")]
pub mod grpc;
+use std::collections::BTreeMap;
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::path::PathBuf;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
-use concat_api::{Api, EventSink};
+use concat_api::{Api, EventSink, Stopper};
pub use hub::{Hub, Subscriber};
+pub use token::{MIN_BITS as MIN_TOKEN_BITS, MIN_CHARS as MIN_TOKEN_CHARS};
+
+/// A fresh token: 128 bits from the operating system's randomness, as 32
+/// hex digits. What a front end offers when asked for a new one.
+pub fn mint_token() -> Result<String, String> {
+ token::mint()
+}
+
+/// Refuses a chosen token that is short or repetitive enough to guess;
+/// the sentence says why. A minted token always passes.
+pub fn check_token(token: &str) -> Result<(), String> {
+ token::check(token)
+}
+
+/// Whether listening on `address` lets other machines in: anything but a
+/// loopback address. A front end warns when this is true; the server
+/// itself allows it, with a token, as a deliberate choice.
+pub fn is_exposed(address: &SocketAddr) -> bool {
+ !address.ip().is_loopback()
+}
+
+/// How long a stop waits on the connection that wakes a listener.
+const WAKE_PATIENCE: std::time::Duration = std::time::Duration::from_secs(2);
+
+/// The address a local caller connects to for a listener bound to
+/// `bound`: the same, except that the unspecified address is loopback.
+pub fn loopback(bound: SocketAddr) -> SocketAddr {
+ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
+ match bound.ip() {
+ IpAddr::V4(ip) if ip.is_unspecified() => {
+ SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), bound.port())
+ }
+ IpAddr::V6(ip) if ip.is_unspecified() => {
+ SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), bound.port())
+ }
+ _ => bound,
+ }
+}
/// Where to listen, and who may connect.
#[derive(Clone, Debug, Default)]
@@ -67,7 +111,7 @@ pub struct Config {
/// empty means the server mints one at start, which only the process
/// that started it can learn, through [`Server::token`]; that is the
/// point, and it is why a bind off loopback needs no token set here
- /// to be safe to make.
+ /// to be safe to make. One that is set must pass [`check_token`].
pub token: Option<String>,
/// The folders the API may write under: a created project, an
/// instantiated template, an export, a preview file. Empty means the
@@ -86,7 +130,8 @@ pub fn home_root() -> Vec<PathBuf> {
}
impl Config {
- /// Refuses a configuration this build or platform cannot serve.
+ /// Refuses a configuration this build or platform cannot serve, or a
+ /// token too weak to stand in front of it.
pub fn check(&self) -> Result<(), String> {
#[cfg(not(unix))]
if self.socket.is_some() {
@@ -98,6 +143,9 @@ impl Config {
"this build has no gRPC: build concat-server with --features grpc".to_owned(),
);
}
+ if let Some(token) = self.token.as_deref().filter(|token| !token.is_empty()) {
+ check_token(token)?;
+ }
Ok(())
}
}
@@ -106,6 +154,7 @@ impl Config {
pub struct Server {
hub: Hub,
dispatcher: Option<JoinHandle<()>>,
+ stopper: Stopper,
json: Option<SocketAddr>,
socket: Option<PathBuf>,
grpc: Option<SocketAddr>,
@@ -114,11 +163,14 @@ pub struct Server {
stop: Arc<AtomicBool>,
listeners: Vec<JoinHandle<()>>,
connections: Connections,
+ seats: Arc<AtomicUsize>,
+ pending: Arc<AtomicUsize>,
}
-/// Every open connection's way of being closed, so a stop reaches the
-/// threads blocked reading them.
-pub(crate) type Connections = Arc<Mutex<Vec<Box<dyn Fn() + Send>>>>;
+/// Every connection that is in, by number, with its way of being closed,
+/// so a stop reaches the threads blocked reading them. A connection takes
+/// itself out when it ends.
+pub(crate) type Connections = Arc<Mutex<BTreeMap<u64, Box<dyn Fn() + Send>>>>;
impl Server {
/// Binds every address in `config` and starts serving. `make` builds
@@ -148,7 +200,7 @@ impl Server {
config.roots.clone()
};
let confined = roots.clone();
- let (hub, dispatcher) = Hub::start(move |events| {
+ let (hub, dispatcher, stopper) = Hub::start(move |events| {
let mut api = make(events)?;
for transport in transports {
api.add_capability(transport);
@@ -157,11 +209,24 @@ impl Server {
Ok(api)
})?;
let stop = Arc::new(AtomicBool::new(false));
- let connections: Connections = Arc::new(Mutex::new(Vec::new()));
- let seats = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let connections: Connections = Arc::new(Mutex::new(BTreeMap::new()));
+ let throttle = Arc::new(throttle::Throttle::default());
+ let door = json::Door {
+ hub: hub.clone(),
+ token: token.clone(),
+ stop: Arc::clone(&stop),
+ connections: Arc::clone(&connections),
+ seats: Arc::new(AtomicUsize::new(0)),
+ pending: Arc::new(AtomicUsize::new(0)),
+ throttle: Arc::clone(&throttle),
+ next_id: Arc::new(AtomicU64::new(1)),
+ };
+ // Built before the first bind, so a bind that fails drops a server
+ // that shuts down what it had already started.
let mut server = Server {
hub,
dispatcher: Some(dispatcher),
+ stopper,
json: None,
socket: None,
grpc: None,
@@ -170,40 +235,37 @@ impl Server {
stop: Arc::clone(&stop),
listeners: Vec::new(),
connections: Arc::clone(&connections),
+ seats: Arc::clone(&door.seats),
+ pending: Arc::clone(&door.pending),
+ };
+ let could_not = |what: &str, error: &dyn std::fmt::Display| {
+ format!("could not listen on {what}: {error}")
};
if let Some(address) = config.json {
let listener = TcpListener::bind(address)
- .map_err(|error| format!("could not listen on {address}: {error}"))?;
+ .map_err(|error| could_not(&address.to_string(), &error))?;
server.json = Some(
listener
.local_addr()
- .map_err(|error| format!("could not listen on {address}: {error}"))?,
+ .map_err(|error| could_not(&address.to_string(), &error))?,
+ );
+ server.listeners.push(
+ json::serve_tcp(listener, door.clone())
+ .map_err(|error| could_not(&address.to_string(), &error))?,
);
- server.listeners.push(json::serve_tcp(
- listener,
- server.hub.clone(),
- server.token.clone(),
- Arc::clone(&stop),
- Arc::clone(&connections),
- Arc::clone(&seats),
- ));
}
#[cfg(unix)]
if let Some(path) = config.socket {
+ let shown = path.display().to_string();
let _ = std::fs::remove_file(&path);
let listener = std::os::unix::net::UnixListener::bind(&path)
- .map_err(|error| format!("could not listen on {}: {error}", path.display()))?;
- server.listeners.push(json::serve_unix(
- listener,
- server.hub.clone(),
- server.token.clone(),
- Arc::clone(&stop),
- Arc::clone(&connections),
- Arc::clone(&seats),
- ));
+ .map_err(|error| could_not(&shown, &error))?;
server.socket = Some(path);
+ server.listeners.push(
+ json::serve_unix(listener, door.clone()).map_err(|error| could_not(&shown, &error))?,
+ );
}
#[cfg(feature = "grpc")]
@@ -213,10 +275,12 @@ impl Server {
server.hub.clone(),
server.token.clone(),
Arc::clone(&stop),
+ Arc::clone(&throttle),
)?;
server.grpc = Some(bound);
server.listeners.push(thread);
}
+ let _ = throttle;
Ok(server)
}
@@ -253,22 +317,58 @@ impl Server {
&self.roots
}
- /// How many callers are connected.
+ /// How many JSON callers are in: connected, with the token presented.
pub fn connections(&self) -> usize {
- self.hub.subscribers()
+ self.seats.load(Ordering::SeqCst)
+ }
+
+ /// How many JSON connections are waiting to present their token.
+ pub fn pending(&self) -> usize {
+ self.pending.load(Ordering::SeqCst)
}
- /// Closes every listener and connection, waits for the jobs still
- /// running, and returns when the API's thread has ended.
+ /// How many connections a stop would have to close.
+ #[cfg(test)]
+ pub(crate) fn registered(&self) -> usize {
+ self.connections.lock().map(|list| list.len()).unwrap_or(0)
+ }
+
+ /// Stops the server and returns once everything it started has ended:
+ /// the listeners and connections closed, the API's own jobs cancelled
+ /// and their threads finished, the API's thread gone.
pub fn stop(mut self) {
- self.shut();
+ self.close_doors();
+ self.finish();
+ }
+
+ /// Stops the server without waiting for the API's thread: when this
+ /// returns the listeners are closed - the address may be bound again
+ /// at once - every connection is hung up and the API's jobs are told
+ /// to stop; the call in flight, if one is, and the jobs' last frame
+ /// are waited for on a thread of their own. For a window, whose event
+ /// loop must not wait on a render.
+ pub fn stop_detached(mut self) {
+ self.close_doors();
+ let dispatcher = self.dispatcher.take();
+ let _ = std::thread::Builder::new()
+ .name("concat server stop".to_owned())
+ .spawn(move || {
+ if let Some(dispatcher) = dispatcher {
+ let _ = dispatcher.join();
+ }
+ });
}
- fn shut(&mut self) {
+ /// Everything a stop does that cannot keep the caller waiting long:
+ /// the listeners joined, the connections hung up, the jobs cancelled
+ /// and the queue closed.
+ fn close_doors(&mut self) {
self.stop.store(true, Ordering::SeqCst);
- // A blocked accept wakes for a connection; make one.
+ // A blocked accept wakes for a connection; make one - to loopback
+ // when the bind was to every interface, which Windows refuses to
+ // connect to and which would leave the join below waiting for ever.
if let Some(address) = self.json {
- let _ = TcpStream::connect(address);
+ let _ = TcpStream::connect_timeout(&loopback(address), WAKE_PATIENCE);
}
#[cfg(unix)]
if let Some(path) = &self.socket {
@@ -276,13 +376,13 @@ impl Server {
}
#[cfg(feature = "grpc")]
if let Some(address) = self.grpc {
- let _ = TcpStream::connect(address);
+ let _ = TcpStream::connect_timeout(&loopback(address), WAKE_PATIENCE);
}
for thread in self.listeners.drain(..) {
let _ = thread.join();
}
if let Ok(mut connections) = self.connections.lock() {
- for close in connections.drain(..) {
+ for (_, close) in std::mem::take(&mut *connections) {
close();
}
}
@@ -290,7 +390,14 @@ impl Server {
if let Some(path) = self.socket.take() {
let _ = std::fs::remove_file(path);
}
+ // Before the queue closes: the API's thread waits for its jobs as
+ // it ends, and a job told to stop ends at its next frame instead
+ // of its last.
+ self.stopper.cancel_all();
self.hub.close();
+ }
+
+ fn finish(&mut self) {
if let Some(dispatcher) = self.dispatcher.take() {
let _ = dispatcher.join();
}
@@ -300,7 +407,8 @@ impl Server {
impl Drop for Server {
fn drop(&mut self) {
if self.dispatcher.is_some() {
- self.shut();
+ self.close_doors();
+ self.finish();
}
}
}
@@ -310,6 +418,9 @@ pub(crate) mod tests {
use super::*;
use concat_api::AppDirs;
+ /// A token strong enough to be accepted, for tests that choose one.
+ pub(crate) const TOKEN: &str = "a test token, long and varied 7f3Q";
+
/// A server over a scratch home, on loopback, on any free port.
pub(crate) fn server(token: Option<&str>) -> (Server, tempfile::TempDir) {
let scratch = tempfile::tempdir().expect("scratch");
@@ -330,8 +441,8 @@ pub(crate) mod tests {
#[test]
fn a_server_keeps_the_token_it_is_given_and_mints_one_otherwise() {
- let (given, _scratch) = server(Some("open sesame"));
- assert_eq!(given.token(), "open sesame");
+ let (given, _scratch) = server(Some(TOKEN));
+ assert_eq!(given.token(), TOKEN);
let (minted, _scratch) = server(None);
assert_eq!(minted.token().len(), 32);
assert!(minted.token().bytes().all(|byte| byte.is_ascii_hexdigit()));
@@ -340,6 +451,35 @@ pub(crate) mod tests {
assert_ne!(empty.token(), minted.token());
}
+ #[test]
+ fn a_weak_token_is_refused_before_anything_listens() {
+ let scratch = tempfile::tempdir().expect("scratch");
+ let dirs = AppDirs {
+ config: scratch.path().join("config"),
+ data: scratch.path().join("data"),
+ };
+ let config = Config {
+ json: Some("127.0.0.1:0".parse().expect("an address")),
+ token: Some("open sesame".to_owned()),
+ ..Config::default()
+ };
+ let refused = Server::start(config, move |events| Ok(Api::with_dirs(dirs, events)));
+ let Err(why) = refused else {
+ panic!("a weak token was taken");
+ };
+ assert!(why.contains("at least"), "{why}");
+ }
+
+ #[test]
+ fn exposure_is_anything_off_loopback() {
+ let exposed = |text: &str| is_exposed(&text.parse().expect("an address"));
+ assert!(!exposed("127.0.0.1:7420"));
+ assert!(!exposed("[::1]:7420"));
+ assert!(exposed("0.0.0.0:7420"));
+ assert!(exposed("[::]:7420"));
+ assert!(exposed("192.168.1.20:7420"));
+ }
+
#[test]
fn a_server_stops_cleanly_with_nobody_connected() {
let (server, _scratch) = server(None);
@@ -347,4 +487,33 @@ pub(crate) mod tests {
assert_eq!(server.connections(), 0);
server.stop();
}
+
+ #[test]
+ fn a_server_on_every_interface_stops_too() {
+ let scratch = tempfile::tempdir().expect("scratch");
+ let dirs = AppDirs {
+ config: scratch.path().join("config"),
+ data: scratch.path().join("data"),
+ };
+ let config = Config {
+ json: Some("0.0.0.0:0".parse().expect("an address")),
+ roots: vec![scratch.path().to_path_buf()],
+ ..Config::default()
+ };
+ let server =
+ Server::start(config, move |events| Ok(Api::with_dirs(dirs, events))).expect("starts");
+ let bound = server.json_addr().expect("listening");
+ assert!(is_exposed(&bound));
+ assert!(loopback(bound).ip().is_loopback());
+ server.stop();
+ }
+
+ #[test]
+ fn a_detached_stop_frees_the_address_at_once() {
+ let (server, _scratch) = server(None);
+ let address = server.json_addr().expect("listening");
+ server.stop_detached();
+ let again = TcpListener::bind(address);
+ assert!(again.is_ok(), "{again:?}");
+ }
}
diff --git a/src/crates/concat-server/src/throttle.rs b/src/crates/concat-server/src/throttle.rs
new file mode 100644
index 00000000..4b75aa9e
--- /dev/null
+++ b/src/crates/concat-server/src/throttle.rs
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Who has presented a wrong token lately, and how long each must wait.
+//!
+//! A minted token is 128 bits and no throttle is needed to keep it; a
+//! chosen one passed [`crate::token::check`] and is still something a
+//! person typed. So a caller gets a few honest mistakes, and after those
+//! every wrong token doubles the wait before that address is heard again,
+//! up to a minute. The right token clears the slate. The book is bounded:
+//! past [`MAX_PEERS`] the quiet entries go first.
+
+use std::collections::HashMap;
+use std::net::IpAddr;
+use std::sync::Mutex;
+use std::time::{Duration, Instant};
+
+/// Wrong tokens forgiven before the waiting starts.
+pub const FREE_FAILURES: u32 = 3;
+/// The first wait, doubled at each failure after it.
+pub const FIRST_WAIT: Duration = Duration::from_millis(250);
+/// The longest wait.
+pub const LONGEST_WAIT: Duration = Duration::from_secs(60);
+/// How many addresses are remembered at once.
+const MAX_PEERS: usize = 4096;
+/// How long an address that stopped failing is remembered.
+const MEMORY: Duration = Duration::from_secs(600);
+
+/// Where a connection came from, as far as the throttle cares: an
+/// address, or this machine through a Unix socket.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
+pub enum Peer {
+ /// A TCP caller's address. The port is not part of it: a new
+ /// connection is a new port.
+ Ip(IpAddr),
+ /// A Unix socket caller, whose access the socket file's permissions
+ /// already decide.
+ Local,
+}
+
+#[derive(Clone, Copy)]
+struct Strikes {
+ failures: u32,
+ until: Instant,
+ last: Instant,
+}
+
+/// The book of recent failures. Shared by every transport of a server.
+#[derive(Default)]
+pub struct Throttle {
+ peers: Mutex<HashMap<Peer, Strikes>>,
+}
+
+impl Throttle {
+ /// How long `peer` must still wait before a token from it is read;
+ /// `None` when it need not.
+ pub fn wait(&self, peer: Peer) -> Option<Duration> {
+ let peers = self.peers.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+ let strikes = peers.get(&peer)?;
+ strikes
+ .until
+ .checked_duration_since(Instant::now())
+ .filter(|left| !left.is_zero())
+ }
+
+ /// `peer` presented a wrong token.
+ pub fn failed(&self, peer: Peer) {
+ let now = Instant::now();
+ let mut peers = self.peers.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+ if peers.len() >= MAX_PEERS && !peers.contains_key(&peer) {
+ peers.retain(|_, strikes| now.duration_since(strikes.last) < MEMORY);
+ if peers.len() >= MAX_PEERS {
+ // Every entry is fresh: an address flood. Forget the one
+ // that has waited longest rather than grow.
+ if let Some(oldest) = peers
+ .iter()
+ .min_by_key(|(_, strikes)| strikes.last)
+ .map(|(peer, _)| *peer)
+ {
+ peers.remove(&oldest);
+ }
+ }
+ }
+ let strikes = peers.entry(peer).or_insert(Strikes {
+ failures: 0,
+ until: now,
+ last: now,
+ });
+ strikes.failures = strikes.failures.saturating_add(1);
+ strikes.last = now;
+ strikes.until = now + wait_after(strikes.failures);
+ }
+
+ /// `peer` presented the right token: its slate is clean.
+ pub fn succeeded(&self, peer: Peer) {
+ self.peers
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .remove(&peer);
+ }
+}
+
+/// The wait after the `failures`th wrong token in a row.
+pub fn wait_after(failures: u32) -> Duration {
+ if failures <= FREE_FAILURES {
+ return Duration::ZERO;
+ }
+ let doublings = (failures - FREE_FAILURES - 1).min(16);
+ FIRST_WAIT
+ .saturating_mul(1 << doublings)
+ .min(LONGEST_WAIT)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_wait_starts_after_the_free_failures_and_doubles_to_a_ceiling() {
+ assert_eq!(wait_after(1), Duration::ZERO);
+ assert_eq!(wait_after(FREE_FAILURES), Duration::ZERO);
+ assert_eq!(wait_after(FREE_FAILURES + 1), FIRST_WAIT);
+ assert_eq!(wait_after(FREE_FAILURES + 2), FIRST_WAIT * 2);
+ assert_eq!(wait_after(FREE_FAILURES + 3), FIRST_WAIT * 4);
+ assert_eq!(wait_after(1000), LONGEST_WAIT);
+ }
+
+ #[test]
+ fn a_peer_waits_after_its_failures_and_the_right_token_clears_it() {
+ let throttle = Throttle::default();
+ let peer = Peer::Ip("192.0.2.1".parse().expect("an address"));
+ let other = Peer::Ip("192.0.2.2".parse().expect("an address"));
+ for _ in 0..FREE_FAILURES {
+ throttle.failed(peer);
+ assert_eq!(throttle.wait(peer), None, "forgiven");
+ }
+ throttle.failed(peer);
+ assert!(throttle.wait(peer).is_some());
+ assert_eq!(throttle.wait(other), None, "only the one that failed");
+ throttle.succeeded(peer);
+ assert_eq!(throttle.wait(peer), None);
+ }
+}
diff --git a/src/crates/concat-server/src/token.rs b/src/crates/concat-server/src/token.rs
index be1ba243..9635eb1b 100644
--- a/src/crates/concat-server/src/token.rs
+++ b/src/crates/concat-server/src/token.rs
@@ -9,16 +9,68 @@
//! refused. A library would do; this is short enough to read, and its
//! test says what it promises.
+use std::collections::HashMap;
use std::hint::black_box;
+/// The fewest characters a chosen token may have.
+pub const MIN_CHARS: usize = 20;
+/// The least a chosen token may be worth, in bits, by [`estimated_bits`].
+pub const MIN_BITS: f64 = 64.0;
+
/// A fresh token: 128 bits from the operating system's randomness, as
/// 32 hex digits.
-pub(crate) fn mint() -> Result<String, String> {
+pub fn mint() -> Result<String, String> {
let mut bytes = [0u8; 16];
getrandom::fill(&mut bytes).map_err(|error| format!("could not mint a token: {error}"))?;
Ok(bytes.iter().map(|byte| format!("{byte:02x}")).collect())
}
+/// Refuses a token a person could guess or a script could walk through:
+/// fewer than [`MIN_CHARS`] characters, or fewer than [`MIN_BITS`] by
+/// [`estimated_bits`]. A minted token passes by a wide margin; "secret",
+/// "aaaaaaaaaaaaaaaaaaaaaaaa" and "abababababababababab" do not.
+pub fn check(token: &str) -> Result<(), String> {
+ let chars = token.chars().count();
+ if chars < MIN_CHARS {
+ return Err(format!(
+ "the token is {chars} characters; a token needs at least {MIN_CHARS}, or leave it empty and one is made"
+ ));
+ }
+ if estimated_bits(token) < MIN_BITS {
+ return Err(
+ "the token repeats itself too much to be hard to guess; leave it empty and one is made"
+ .to_owned(),
+ );
+ }
+ Ok(())
+}
+
+/// A pessimistic guess at a token's strength: its length times the
+/// Shannon entropy of its own characters. Repetition costs - a token of
+/// one letter is worth nothing - and no dictionary is consulted, which is
+/// why the floor in [`check`] is set where a random token clears it with
+/// room and a phrase needs to be long.
+pub fn estimated_bits(token: &str) -> f64 {
+ let mut counts: HashMap<char, usize> = HashMap::new();
+ let mut total = 0usize;
+ for char in token.chars() {
+ *counts.entry(char).or_default() += 1;
+ total += 1;
+ }
+ if total == 0 {
+ return 0.0;
+ }
+ let total = total as f64;
+ let per_char: f64 = counts
+ .values()
+ .map(|&count| {
+ let share = count as f64 / total;
+ -share * share.log2()
+ })
+ .sum();
+ per_char * total
+}
+
/// Whether `presented` is `expected`, in time that depends on the length
/// of `expected` and on nothing about where the two differ.
///
@@ -48,6 +100,27 @@ mod tests {
assert_ne!(first, second);
}
+ #[test]
+ fn a_weak_token_is_refused_and_a_minted_one_is_not() {
+ for _ in 0..64 {
+ let minted = mint().expect("mints");
+ assert!(check(&minted).is_ok(), "{minted}: {}", estimated_bits(&minted));
+ }
+ for weak in [
+ "",
+ "secret",
+ "open sesame",
+ "password12345678",
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "abababababababababababababab",
+ "abcabcabcabcabcabcabc",
+ ] {
+ assert!(check(weak).is_err(), "{weak:?} passed");
+ }
+ assert!(check("correct horse battery staple, purple").is_ok());
+ assert!(check("Xk9#mQ2$vL7&pR4!tW8z").is_ok());
+ }
+
#[test]
fn the_compare_accepts_the_token_and_nothing_near_it() {
assert!(matches(b"open sesame", b"open sesame"));