LordOS — a free, fast and secure GNU/Linux distribution based on Debian, with its own desktop environment: display server and Wayland compositor in C, a React UI engine on QuickJS, and private native apps.
C cdebiandesktop-environmentebpffirewalllandlocklinuxlinux-distributionopen-sourceprivacyquickjsraspberry-pi-5reactsecuritywaylandwayland-compositor
/* lm_stream.c — the remote desktop's live encoder (MK_STREAM, docs/REMOTE.md, "Video (WebRTC)").
*
* One per session and codec: lord-server copies each changed frame of the session (protected windows
* already black) into a free slot of a sealed memfd (MM_INRING / MM_INFRAME), as for the screen
* recording; here it becomes H.264 (VA-API when the card encodes it, else libx264) or VP8 (libvpx),
* tuned for latency: no B-frames, one frame in, one frame out, a keyframe only when asked
* (MM_KEYFRAME: a new viewer or a lost packet) and the bit rate the slowest viewer can take
* (MM_RATE). The session's sound arrives in an audio ring that lord-server fills from PipeWire and
* goes out as Opus (20 ms). Every encoded frame goes back as MM_PACKET chunks; the bridges of the
* session's viewers only add the per-connection encryption. */
#include "lm.h"
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <libavutil/channel_layout.h>
#include <libavutil/hwcontext.h>
#include <libavutil/opt.h>
#include <libswscale/swscale.h>
enum { CODEC_H264 = 1, CODEC_VP8 = 2 };
typedef struct {
AVCodecContext *c;
AVBufferRef *frames; /* VA-API surfaces */
struct SwsContext *sws;
AVFrame *yuv;
AVPacket *pkt;
int w, h, codec, hw, force_key;
const char *name;
unsigned kbps;
} Vid;
typedef struct {
AVCodecContext *c;
AVFrame *f;
AVPacket *pkt;
int16_t buf[960 * 4];
int have;
int64_t pts;
} Aud;
static Vid V;
static Aud A;
/* Blocking send of one MM_PACKET message (the helper may wait; lord-server never does). */
static int send_wait(const int64_t v[6], uint32_t id, const uint8_t *p, size_t n) {
for (int tries = 0; tries < 50; tries++) {
if (!mp_send(LM_SOCK, MM_PACKET, id, v, p, n, NULL, 0)) return 0;
if (errno != EAGAIN && errno != EWOULDBLOCK && errno != ENOBUFS) return -1;
struct pollfd pf = { LM_SOCK, POLLOUT, 0 };
poll(&pf, 1, 20);
}
return -1;
}
/* A packet as chunks of at most MP_PAYLOAD_MAX: v[0] total, v[1] offset, v[2] pts, v[3] flags. */
static void send_packet(uint32_t kind, const uint8_t *d, size_t n, int64_t pts, int key) {
if (!n || n > (8u << 20)) return;
for (size_t off = 0; off < n; off += MP_PAYLOAD_MAX) {
size_t c = n - off > MP_PAYLOAD_MAX ? MP_PAYLOAD_MAX : n - off;
int64_t v[6] = { (int64_t)n, (int64_t)off, pts, key, 0, 0 };
if (send_wait(v, kind, d + off, c)) return;
}
}
static AVCodecContext *open_h264_vaapi(Vid *e, AVBufferRef *hw, int fps) {
const AVCodec *codec = avcodec_find_encoder_by_name("h264_vaapi");
if (!hw || !codec) return NULL;
AVCodecContext *c = avcodec_alloc_context3(codec);
AVBufferRef *fr = av_hwframe_ctx_alloc(hw);
AVHWFramesContext *fc = fr ? (AVHWFramesContext *)fr->data : NULL;
if (!c || !fc) { av_buffer_unref(&fr); avcodec_free_context(&c); return NULL; }
fc->format = AV_PIX_FMT_VAAPI;
fc->sw_format = AV_PIX_FMT_NV12;
fc->width = e->w;
fc->height = e->h;
fc->initial_pool_size = 4;
if (av_hwframe_ctx_init(fr) < 0) { av_buffer_unref(&fr); avcodec_free_context(&c); return NULL; }
c->hw_frames_ctx = av_buffer_ref(fr);
c->pix_fmt = AV_PIX_FMT_VAAPI;
c->width = e->w;
c->height = e->h;
c->time_base = (AVRational){ 1, 1000 };
c->framerate = (AVRational){ fps, 1 };
c->gop_size = fps * 60;
c->max_b_frames = 0;
c->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE;
c->bit_rate = (int64_t)e->kbps * 1000;
c->rc_max_rate = c->bit_rate;
c->rc_buffer_size = (int)(c->bit_rate / 2);
av_opt_set(c->priv_data, "rc_mode", "VBR", 0);
if (avcodec_open2(c, codec, NULL) < 0) { av_buffer_unref(&fr); avcodec_free_context(&c); LM_LOG("VA-API does not encode H.264 here: libx264"); return NULL; }
e->frames = fr;
e->hw = 1;
e->name = "h264_vaapi";
return c;
}
static AVCodecContext *open_sw(Vid *e, const char *name, int fps) {
const AVCodec *codec = avcodec_find_encoder_by_name(name);
if (!codec) return NULL;
AVCodecContext *c = avcodec_alloc_context3(codec);
if (!c) return NULL;
c->width = e->w;
c->height = e->h;
c->pix_fmt = AV_PIX_FMT_YUV420P;
c->time_base = (AVRational){ 1, 1000 };
c->framerate = (AVRational){ fps, 1 };
c->gop_size = fps * 60;
c->max_b_frames = 0;
/* the desktop is sRGB: BT.709 primaries and matrix with the sRGB transfer, so the browser doesn't
* re-shape the tones (dark greys came out lighter than the lossless tiles next to them) */
c->color_range = AVCOL_RANGE_MPEG;
c->colorspace = AVCOL_SPC_BT709;
c->color_primaries = AVCOL_PRI_BT709;
c->color_trc = AVCOL_TRC_IEC61966_2_1;
AVDictionary *o = NULL;
if (e->codec == CODEC_H264) {
c->thread_count = 2;
c->rc_max_rate = (int64_t)e->kbps * 1000;
c->rc_buffer_size = (int)(c->rc_max_rate / 3); /* about 1/3 s: low latency */
av_dict_set(&o, "preset", "ultrafast", 0);
av_dict_set(&o, "tune", "zerolatency", 0);
av_dict_set(&o, "profile", "baseline", 0);
av_dict_set(&o, "crf", "22", 0);
av_dict_set(&o, "forced-idr", "1", 0);
/* ultrafast turns the deblocking filter off, which left video in visible blocks at a few
* Mbit/s; it costs ~5 % of the encoder (1280x800 on a 1.8 GHz Xeon: 88 → 84 fps, 2 threads) */
av_dict_set(&o, "x264-params", "repeat-headers=1:annexb=1:aud=0:scenecut=0:deblock=1", 0);
} else {
c->thread_count = 2;
c->bit_rate = (int64_t)e->kbps * 1000;
c->rc_min_rate = 0;
c->rc_max_rate = c->bit_rate;
c->rc_buffer_size = (int)(c->bit_rate / 3);
c->qmin = 4;
c->qmax = 56;
av_dict_set(&o, "deadline", "realtime", 0);
av_dict_set(&o, "cpu-used", "8", 0);
av_dict_set(&o, "lag-in-frames", "0", 0);
av_dict_set(&o, "error-resilient", "1", 0);
av_dict_set(&o, "static-thresh", "100", 0);
}
int r = avcodec_open2(c, codec, &o);
av_dict_free(&o);
if (r < 0) { avcodec_free_context(&c); return NULL; }
e->name = name;
return c;
}
AVCodecContext *lm_stream_sw_encoder(int vp8, int w, int h, int fps, unsigned kbps) {
Vid e;
memset(&e, 0, sizeof e);
e.codec = vp8 ? CODEC_VP8 : CODEC_H264;
e.w = w & ~1;
e.h = h & ~1;
e.kbps = kbps;
return open_sw(&e, vp8 ? "libvpx" : "libx264", fps);
}
static int vid_open(Vid *e, int codec, int w, int h, int fps, unsigned kbps, AVBufferRef *hw) {
memset(e, 0, sizeof *e);
e->codec = codec;
e->w = w & ~1;
e->h = h & ~1;
e->kbps = kbps;
if (e->w < 16 || e->h < 16) return -1;
if (codec == CODEC_H264) {
e->c = open_h264_vaapi(e, hw, fps);
if (!e->c) e->c = open_sw(e, "libx264", fps);
} else e->c = open_sw(e, "libvpx", fps);
if (!e->c) return -1;
e->pkt = av_packet_alloc();
e->yuv = av_frame_alloc();
e->yuv->format = e->frames ? AV_PIX_FMT_NV12 : AV_PIX_FMT_YUV420P;
e->yuv->width = e->w;
e->yuv->height = e->h;
if (av_frame_get_buffer(e->yuv, 0) < 0) return -1;
LM_LOG("remote desktop video: %dx%d with %s, up to %u kbit/s", e->w, e->h, e->name, kbps);
return 0;
}
static void vid_close(Vid *e) {
avcodec_free_context(&e->c);
av_buffer_unref(&e->frames);
sws_freeContext(e->sws);
av_frame_free(&e->yuv);
av_packet_free(&e->pkt);
}
static void vid_drain(Vid *e) {
while (avcodec_receive_packet(e->c, e->pkt) >= 0) {
send_packet(0, e->pkt->data, (size_t)e->pkt->size, e->pkt->pts, (e->pkt->flags & AV_PKT_FLAG_KEY) != 0);
av_packet_unref(e->pkt);
}
}
static int vid_frame(Vid *e, const uint8_t *bgra, int w, int h, int64_t ms) {
if (!e->sws) {
e->sws = sws_getContext(w, h, AV_PIX_FMT_BGRA, e->w, e->h, e->yuv->format, SWS_POINT, NULL, NULL, NULL);
if (!e->sws) return -1;
sws_setColorspaceDetails(e->sws, sws_getCoefficients(SWS_CS_ITU709), 1, sws_getCoefficients(SWS_CS_ITU709), 0, 0, 1 << 16, 1 << 16);
}
if (av_frame_make_writable(e->yuv) < 0) return -1;
const uint8_t *src[1] = { bgra };
int stride[1] = { w * 4 };
sws_scale(e->sws, src, stride, 0, h, e->yuv->data, e->yuv->linesize);
AVFrame *in = e->yuv, *hwf = NULL;
if (e->frames) {
hwf = av_frame_alloc();
if (av_hwframe_get_buffer(e->frames, hwf, 0) < 0 || av_hwframe_transfer_data(hwf, e->yuv, 0) < 0) { av_frame_free(&hwf); return -1; }
in = hwf;
}
in->pts = ms;
in->pict_type = e->force_key ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
if (e->force_key) in->flags |= AV_FRAME_FLAG_KEY; else in->flags &= ~AV_FRAME_FLAG_KEY;
e->force_key = 0;
int r = avcodec_send_frame(e->c, in);
av_frame_free(&hwf);
if (r < 0) return r;
vid_drain(e);
return 0;
}
static void vid_rate(Vid *e, unsigned kbps) {
if (!e->c || kbps == e->kbps) return;
e->kbps = kbps;
/* libx264 reconfigures itself when these change between frames; the others keep their rate */
if (e->codec == CODEC_H264 && !e->hw) {
e->c->rc_max_rate = (int64_t)kbps * 1000;
e->c->rc_buffer_size = (int)(e->c->rc_max_rate / 3);
}
}
static int aud_open(Aud *a) {
const AVCodec *codec = avcodec_find_encoder_by_name("libopus");
if (!codec) { LM_LOG("no Opus encoder: the remote desktop has no sound"); return -1; }
a->c = avcodec_alloc_context3(codec);
a->c->sample_fmt = AV_SAMPLE_FMT_S16;
a->c->sample_rate = 48000;
av_channel_layout_default(&a->c->ch_layout, 1);
a->c->bit_rate = 64000;
a->c->time_base = (AVRational){ 1, 48000 };
AVDictionary *o = NULL;
av_dict_set(&o, "application", "lowdelay", 0);
av_dict_set(&o, "frame_duration", "20", 0);
int r = avcodec_open2(a->c, codec, &o);
av_dict_free(&o);
if (r < 0) { avcodec_free_context(&a->c); return -1; }
a->f = av_frame_alloc();
a->pkt = av_packet_alloc();
return 0;
}
static void on_pcm(const int16_t *s, int k) {
Aud *a = &A;
if (!a->c) return;
while (k > 0) {
int take = 960 - a->have;
if (take > k) take = k;
memcpy(a->buf + a->have, s, (size_t)take * 2);
a->have += take;
s += take;
k -= take;
if (a->have < 960) break;
a->f->nb_samples = 960;
a->f->format = AV_SAMPLE_FMT_S16;
a->f->sample_rate = 48000;
av_channel_layout_default(&a->f->ch_layout, 1);
if (av_frame_get_buffer(a->f, 0) < 0) return;
memcpy(a->f->data[0], a->buf, 960 * 2);
a->f->pts = a->pts;
a->pts += 960;
a->have = 0;
if (avcodec_send_frame(a->c, a->f) >= 0)
while (avcodec_receive_packet(a->c, a->pkt) >= 0) {
send_packet(1, a->pkt->data, (size_t)a->pkt->size, a->pkt->pts, 0);
av_packet_unref(a->pkt);
}
av_frame_unref(a->f);
}
}
void lm_stream(const char *params, int ring_fd, int render) {
char cname[16] = "";
mp_param(params, "codec", cname, sizeof cname);
int codec = !strcmp(cname, "vp8") ? CODEC_VP8 : !strcmp(cname, "h264") ? CODEC_H264 : 0;
int w = (int)mp_param_int(params, "width", 0), h = (int)mp_param_int(params, "height", 0);
int fps = (int)mp_param_int(params, "fps", 30);
long kbps = mp_param_int(params, "kbps", 4000);
if (fps < 5 || fps > 240) fps = 30;
if (kbps < 200 || kbps > 50000) kbps = 4000;
char accel[24];
AVBufferRef *hw = codec == CODEC_H264 ? lm_hw_device(lm_hw_accel(params, accel, sizeof accel), render) : NULL;
if (!codec || w < 16 || h < 16 || w > MP_MAX_DIM || h > MP_MAX_DIM || vid_open(&V, codec, w, h, fps, (unsigned)kbps, hw)) {
lm_error("media.noVideoEncoder");
mp_sendv(LM_SOCK, MM_STATE, 0, MS_FAILED, 0, 0, 0);
return;
}
size_t ring_len = 0;
void *ring = lm_mic_ring(ring_fd, &ring_len);
if (ring_fd >= 0) close(ring_fd);
if (ring && aud_open(&A)) { munmap(ring, ring_len); ring = NULL; }
mp_sendv(LM_SOCK, MM_STATE, 0, MS_RECORDING, V.hw, 0, 0);
static char payload[MP_PAYLOAD_MAX + 1];
uint8_t *rmap = NULL;
size_t ring_bytes = 0;
int rw = 0, rh = 0, slots = 0, stop = 0;
long frames = 0;
double enc_ms = 0, last_stats = lm_now_ms();
while (!stop) {
struct pollfd pf[1] = { { LM_SOCK, POLLIN, 0 } };
if (poll(pf, 1, ring ? 10 : 200) < 0 && errno != EINTR) break;
if (pf[0].revents & (POLLHUP | POLLERR)) break;
if (pf[0].revents & POLLIN) {
MediaMsg m;
int fds[MP_MAX_FDS], r;
while ((r = mp_recv(LM_SOCK, &m, payload, sizeof payload, fds, 0)) != -1) {
if (r == 0 || m.type == MM_STOP) { stop = 1; break; }
if (m.type == MM_INRING && fds[0] >= 0 && !rmap) {
struct stat st;
rw = (int)m.v[1]; rh = (int)m.v[2]; slots = (int)m.v[0];
ring_bytes = (size_t)rw * rh * 4 * (size_t)slots;
if (rw == w && rh == h && slots > 0 && slots <= 4 && !fstat(fds[0], &st) && (size_t)st.st_size >= ring_bytes &&
(fcntl(fds[0], F_GET_SEALS) & F_SEAL_SHRINK)) {
void *mm = mmap(NULL, ring_bytes, PROT_READ, MAP_SHARED, fds[0], 0);
if (mm != MAP_FAILED) rmap = mm;
}
} else if (m.type == MM_INFRAME && rmap && m.id < (uint32_t)slots) {
double t0 = lm_now_ms();
if (vid_frame(&V, rmap + (size_t)m.id * rw * rh * 4, rw, rh, m.v[0]) >= 0) frames++;
enc_ms += lm_now_ms() - t0;
mp_sendv(LM_SOCK, MM_SLOTFREE, m.id, 0, 0, 0, 0);
} else if (m.type == MM_KEYFRAME) V.force_key = 1;
else if (m.type == MM_RATE && m.v[0] >= 200 && m.v[0] <= 50000) vid_rate(&V, (unsigned)m.v[0]);
for (int i = 0; i < MP_MAX_FDS; i++) if (fds[i] >= 0) close(fds[i]);
}
}
if (ring) lm_mic_drain(ring, on_pcm);
if (lm_now_ms() - last_stats >= 2000) {
mp_sendv(LM_SOCK, MM_STATS, 0, frames, frames ? (int64_t)(enc_ms * 100 / frames) : 0, V.hw, V.kbps);
last_stats = lm_now_ms();
}
}
LM_LOG("remote desktop video: %ld frames, %.2f ms each with %s", frames, frames ? enc_ms / frames : 0.0, V.name);
vid_close(&V);
if (A.c) { avcodec_free_context(&A.c); av_frame_free(&A.f); av_packet_free(&A.pkt); }
if (ring) munmap(ring, ring_len);
if (rmap) munmap(rmap, ring_bytes);
mp_sendv(LM_SOCK, MM_STATE, 0, MS_DONE, 0, 0, 0);
}