LordOS — a free, fast and secure GNU/Linux distribution based on Debian, with its own desktop environment: display server and Wayland compositor in C, a React UI engine on QuickJS, and private native apps.
C cdebiandesktop-environmentebpffirewalllandlocklinuxlinux-distributionopen-sourceprivacyquickjsraspberry-pi-5reactsecuritywaylandwayland-compositor
Remote desktop: compare only the tiles the compositor drew over (all of them twice a second anyway), copy only what changed into each encoder slot, and read a monitorless session's scene directly instead of copying it every frame
Federico Pereira9e42a14f859f
diff --git a/docs/REMOTE.md b/docs/REMOTE.mdindex ff1c369..d1d91c1 100644--- a/docs/REMOTE.md+++ b/docs/REMOTE.md@@ -267,6 +267,12 @@ screen's refresh rate, and whether the session composes on the GPU or the CPU (htime per frame); 📌 keeps the bar shown (remembered in that browser). The session sends it as `RP_VINFO` (fixed-size numbers, when they change), the bridgeturns it into `{"t":"vinfo"}` with numbers only.+**Only what changed costs anything.** The compositor tells the mirror where it drew (`mirror_frame`'s+damage box): only those 64×64 tiles are compared with the last frame (all of them twice a second+anyway, in case), and each of the encoder's three input slots copies only what changed since it was+last written. A session without a monitor reads the scene directly (no copy) unless a fullscreen app+paints over it.+**The GPU keeps composing.** Until 0.8.3 a remote viewer switched the session to CPU composition.Now `gpu_frame` reads the composed frame back (`gpu_read_next`) once, for the encoder and the tiles,and the screen keeps its GPU composition. Only two cases still use the CPU path: protected windows ondiff --git a/server/src/main.c b/server/src/main.cindex 000be29..d0c9e77 100644--- a/server/src/main.c+++ b/server/src/main.c@@ -1243,7 +1243,10 @@ static void present_cpu(Server *s, Out *o, int index, Rect r, int feed) {const Canvas *seen = media_composed(&out, mirror_watchers() > 0, &mc);view_reset();mirror_composed_on_gpu(0); /* this frame was painted on the CPU */- mirror_frame(seen->px, seen->stride, W, H);+ /* only what went into this buffer can differ from the last frame (not with a fullscreen app+ * painted over it, nor when the frame is the copy with protected windows in black) */+ int dmg[4] = { need.x0, need.y0, need.x1, need.y1 };+ mirror_frame(seen->px, seen->stride, W, H, seen == &out && !wl_fullscreen_active() ? dmg : NULL);mirror_cursor(px_, py_, k.mode, k.px, k.w, k.h, k.hx, k.hy, k.gen);}cursor_local(s, &k, out_scale(o)); /* the arrow is also an image (cursor.c) */@@ -1283,11 +1286,22 @@ static void present_cpu(Server *s, Out *o, int index, Rect r, int feed) {}/* Memory frames (headless session, the capture view): scene + fullscreen, then the remote desktop. */-static void present_memory(Server *s, Out *o, int index, int feed) {+static void present_memory(Server *s, Out *o, int index, int feed, Rect drawn) {if (!o->frame) return;+ /* Without a fullscreen app the frame is the scene itself: read it there, no copy. A fullscreen app+ * paints over a copy (o->frame). Right after one, the whole screen counts as changed. */+ static int fs_before;+ int fs = wl_fullscreen_active(), whole = fs || fs_before;+ fs_before = fs;Canvas out;- canvas_init(&out, o->frame, o->scene.w, o->scene.h, o->scene.w);- canvas_copy(&out, &o->scene);+ if (fs) {+ canvas_init(&out, o->frame, o->scene.w, o->scene.h, o->scene.w);+ canvas_copy(&out, &o->scene);+ } else out = o->scene;+ if (drawn.x0 < 0) drawn.x0 = 0;+ if (drawn.y0 < 0) drawn.y0 = 0;+ if (drawn.x1 > out.w) drawn.x1 = out.w;+ if (drawn.y1 > out.h) drawn.y1 = out.h;view_of(o, index);wl_paint_fullscreen(&out);Cur k;@@ -1299,7 +1313,8 @@ static void present_memory(Server *s, Out *o, int index, int feed) {MediaCursor mc = { x, y, k.mode, k.px, k.w, k.h, k.hx, k.hy };const Canvas *seen = media_composed(&out, mirror_watchers() > 0, &mc);mirror_composed_on_gpu(0); /* this frame was painted on the CPU */- mirror_frame(seen->px, seen->stride, o->scene.w, o->scene.h);+ int dmg[4] = { drawn.x0, drawn.y0, drawn.x1, drawn.y1 };+ mirror_frame(seen->px, seen->stride, o->scene.w, o->scene.h, seen == &out && !whole ? dmg : NULL);mirror_cursor(x, y, k.mode, k.px, k.w, k.h, k.hx, k.hy, k.gen);}view_reset();@@ -1353,8 +1368,8 @@ static void render_frame(Server *s) {int main_shown = 0; /* the main display got this frame */if (s->headless) {Out *o = &s->out[0];- paint_out(s, o, 0, 1);- present_memory(s, o, 0, 1);+ Rect drawn = paint_out(s, o, 0, 1);+ present_memory(s, o, 0, 1, drawn);} else {for (int i = 0; i < OUT_MAX; i++) {Out *o = &s->out[i];@@ -1387,7 +1402,7 @@ static void render_frame(Server *s) {int cx, cy;pointer_phys(s, o, &cx, &cy);mirror_composed_on_gpu(1);- mirror_frame(rb, o->scene.w, o->scene.w, o->scene.h);+ mirror_frame(rb, o->scene.w, o->scene.w, o->scene.h, NULL);mirror_cursor(cx, cy, k.mode, k.px, k.w, k.h, k.hx, k.hy, k.gen);}view_reset();@@ -1413,8 +1428,8 @@ static void render_frame(Server *s) {c->full = out_frames(c, 1) == 0;}if (c->scene.px && (c->full || !box_empty(c->pend) || c->cdirty || wl_fullscreen_active())) {- paint_out(s, c, OUT_MAX, 1);- present_memory(s, c, OUT_MAX, 1);+ Rect drawn = paint_out(s, c, OUT_MAX, 1);+ present_memory(s, c, OUT_MAX, 1, drawn);}} else if (c->on) { out_free(c); c->on = 0; c->pend = (Box){ 0, 0, 0, 0 }; }}diff --git a/server/src/media/mstream.c b/server/src/media/mstream.cindex 7264818..4286e38 100644--- a/server/src/media/mstream.c+++ b/server/src/media/mstream.c@@ -33,6 +33,7 @@ typedef struct {size_t len, cap, total;long frames, enc100;int hw;+ int dirty[4][4]; /* per input slot: what changed since it was last written (x0, y0, x1, y1) */} Enc;static Enc E[MS_CODECS];@@ -143,6 +144,7 @@ static int start(Enc *e, int codec) {e->w = fw;e->h = fh;e->t0 = now_ms();+ for (int i = 0; i < 4; i++) { e->dirty[i][0] = e->dirty[i][1] = 0; e->dirty[i][2] = fw; e->dirty[i][3] = fh; }e->last_feed = 0;e->pending = 1;e->sent_kbps = e->kbps;@@ -210,8 +212,13 @@ static void feed(Enc *e, int codec) {int slot = -1;for (int i = 0; i < s->in_slots; i++) if (!s->in_busy[i]) { slot = i; break; }if (slot < 0) return; /* the encoder is behind: the newest frame waits */+ /* only what changed since this slot was last written: the rest of it is already the screen */uint32_t *dst = (uint32_t *)(s->in_map + (size_t)slot * e->w * e->h * 4);- for (int y = 0; y < e->h; y++) memcpy(dst + (size_t)y * e->w, fpx + (size_t)y * fstride, (size_t)e->w * 4);+ int *d = slot < 4 ? e->dirty[slot] : NULL, full[4] = { 0, 0, e->w, e->h };+ if (!d) d = full;+ int x0 = d[0] < 0 ? 0 : d[0], y0 = d[1] < 0 ? 0 : d[1], x1 = d[2] > e->w ? e->w : d[2], y1 = d[3] > e->h ? e->h : d[3];+ for (int y = y0; y < y1 && x1 > x0; y++) memcpy(dst + (size_t)y * e->w + x0, fpx + (size_t)y * fstride + x0, (size_t)(x1 - x0) * 4);+ if (slot < 4) d[0] = d[1] = d[2] = d[3] = 0;if (e->kbps != e->sent_kbps) { mp_sendv(s->sock, MM_RATE, 0, e->kbps, 0, 0, 0); e->sent_kbps = e->kbps; }if (e->want_key) { mp_sendv(s->sock, MM_KEYFRAME, 0, 0, 0, 0, 0); e->want_key = 0; }if (mp_sendv(s->sock, MM_INFRAME, (uint32_t)slot, (int64_t)(t - e->t0), 0, 0, 0)) return;@@ -222,15 +229,27 @@ static void feed(Enc *e, int codec) {e->flushing = 0;}-void mstream_frame(const uint32_t *px, int stride, int w, int h) {+void mstream_frame(const uint32_t *px, int stride, int w, int h, const int *box) {fpx = px;fstride = stride;fw = w & ~1;fh = h & ~1;+ int all[4] = { 0, 0, fw, fh };+ if (!box) box = all;for (int c = 1; c < MS_CODECS; c++) {Enc *e = &E[c];if (!e->viewers) continue;if (e->id && (e->w != fw || e->h != fh)) stop(e); /* new screen size: a new encoder */+ for (int i = 0; i < 4; i++) { /* every slot misses this change */+ int *d = e->dirty[i];+ if (d[2] <= d[0] || d[3] <= d[1]) memcpy(d, box, sizeof all);+ else {+ if (box[0] < d[0]) d[0] = box[0];+ if (box[1] < d[1]) d[1] = box[1];+ if (box[2] > d[2]) d[2] = box[2];+ if (box[3] > d[3]) d[3] = box[3];+ }+ }e->pending = 1;feed(e, c);}diff --git a/server/src/media/mstream.h b/server/src/media/mstream.hindex 8279abc..d9b4f8c 100644--- a/server/src/media/mstream.h+++ b/server/src/media/mstream.h@@ -18,7 +18,8 @@ void mstream_init(const MStreamOut *out);/* How many viewers want this codec, with sound, and the lowest bit rate among them. 0 viewers stops it. */void mstream_want(int codec, int viewers, int audio, unsigned kbps, int fps);/* The session's screen changed (the frame stays valid until the next call or mstream_want(…, 0)). */-void mstream_frame(const uint32_t *px, int stride, int w, int h);+/* box: what changed since the last call (x0, y0, x1, y1), or NULL for all of it. */+void mstream_frame(const uint32_t *px, int stride, int w, int h, const int *box);void mstream_keyframe(int codec);void mstream_tick(void);double mstream_timeout(void); /* ms until mstream_tick has work, or -1 */diff --git a/server/src/remote/mirror.c b/server/src/remote/mirror.cindex 0ae0794..a3a4979 100644--- a/server/src/remote/mirror.c+++ b/server/src/remote/mirror.c@@ -304,7 +304,7 @@ static void handle(Client *c, int type, const unsigned char *d, uint32_t n) {c->vfps = hz >= 24 && hz <= 360 ? hz : 30;if (c->vcodec && c->vcodec != was) c->vneed_key = 1;video_update();- if (c->vcodec && fb) mstream_frame(fb, W, W, H);+ if (c->vcodec && fb) mstream_frame(fb, W, W, H, NULL);else if (c->vcodec && cb.refresh) cb.refresh(); /* the first viewer: a whole frame */}break;@@ -571,7 +571,10 @@ void mirror_dispatch(const struct pollfd *p, int n) {}}-void mirror_frame(const uint32_t *px, int stride, int w, int h) {+#define FULL_CHECK_MS 500 /* with damage hints, every tile is still compared this often */++void mirror_frame(const uint32_t *px, int stride, int w, int h, const int *dmg) {+ static double full_at;if (lfd < 0 || !mirror_watchers()) { free(fb); fb = NULL; return; }int resized = !fb || w != W || h != H, any = 0;int new_size = w != W || h != H;@@ -592,8 +595,19 @@ void mirror_frame(const uint32_t *px, int stride, int w, int h) {} else {if (!chg) chg = calloc((size_t)TW * TH, 1);if (chg) memset(chg, 0, (size_t)TW * TH);- for (int ty = 0; ty < TH; ty++)- for (int tx = 0; tx < TW; tx++) {+ /* only the tiles the compositor drew over (the whole screen now and then, in case) */+ int tx0 = 0, ty0 = 0, tx1 = TW, ty1 = TH;+ double t = now_ms();+ if (dmg && t - full_at < FULL_CHECK_MS) {+ tx0 = dmg[0] < 0 ? 0 : dmg[0] / TILE;+ ty0 = dmg[1] < 0 ? 0 : dmg[1] / TILE;+ tx1 = dmg[2] <= 0 ? 0 : (dmg[2] + TILE - 1) / TILE;+ ty1 = dmg[3] <= 0 ? 0 : (dmg[3] + TILE - 1) / TILE;+ if (tx1 > TW) tx1 = TW;+ if (ty1 > TH) ty1 = TH;+ } else full_at = t;+ for (int ty = ty0; ty < ty1; ty++)+ for (int tx = tx0; tx < tx1; tx++) {int x = tx * TILE, y = ty * TILE;int tw = x + TILE > W ? W - x : TILE, th = y + TILE > H ? H - y : TILE;int changed = 0;@@ -612,7 +626,20 @@ void mirror_frame(const uint32_t *px, int stride, int w, int h) {send_damage();int video = 0;for (int i = 0; i < MAX_CLIENTS; i++) if (cl[i].fd >= 0 && cl[i].vcodec) video = 1;- if (video) mstream_frame(fb, W, W, H);+ if (video) {+ /* the changed tiles' box: the encoder's slots copy only that */+ int box[4] = { W, H, 0, 0 };+ for (int ty = 0; chg && ty < TH; ty++)+ for (int tx = 0; tx < TW; tx++) {+ if (!chg[ty * TW + tx]) continue;+ int x = tx * TILE, y = ty * TILE, x1 = x + TILE > W ? W : x + TILE, y1 = y + TILE > H ? H : y + TILE;+ if (x < box[0]) box[0] = x;+ if (y < box[1]) box[1] = y;+ if (x1 > box[2]) box[2] = x1;+ if (y1 > box[3]) box[3] = y1;+ }+ mstream_frame(fb, W, W, H, chg && box[2] > box[0] ? box : NULL);+ }}for (int i = 0; i < MAX_CLIENTS; i++) pump(&cl[i]);}diff --git a/server/src/remote/mirror.h b/server/src/remote/mirror.hindex eec6031..42f1d82 100644--- a/server/src/remote/mirror.h+++ b/server/src/remote/mirror.h@@ -50,8 +50,10 @@ void mirror_greeter(const char *json);void mirror_shown(void);/* A new screen frame (without the cursor), for the encoder; compares it tile by tile with the previous- * one: the damage goes out at once, the lossless tiles once the screen is still. */-void mirror_frame(const uint32_t *px, int stride, int w, int h);+ * one: the damage goes out at once, the lossless tiles once the screen is still. dmg (x0, y0, x1, y1):+ * where the compositor drew since the last call, so only those tiles are compared (all of them twice a+ * second anyway); NULL: anywhere. */+void mirror_frame(const uint32_t *px, int stride, int w, int h, const int *dmg);/* Cursor: position and mode (0 arrow, 1 app image, 2 hidden); the image only if gen changed. */void mirror_cursor(int x, int y, int mode, const uint32_t *argb, int w, int h, int hx, int hy, unsigned gen);/* Lossless tiles for viewers whose video plays once the screen is still, the encoder's