Skip to content

LordOS — a free, fast and secure GNU/Linux distribution based on Debian, with its own desktop environment: display server and Wayland compositor in C, a React UI engine on QuickJS, and private native apps.

Remote desktop: compare only the tiles the compositor drew over (all of them twice a second anyway), copy only what changed into each encoder slot, and read a monitorless session's scene directly instead of copying it every frame

Federico Pereira9e42a14f859f
diff --git a/docs/REMOTE.md b/docs/REMOTE.md
index ff1c369..d1d91c1 100644
--- a/docs/REMOTE.md
+++ b/docs/REMOTE.md
@@ -267,6 +267,12 @@ screen's refresh rate, and whether the session composes on the GPU or the CPU (h
time per frame); 📌 keeps the bar shown (remembered in that browser). The session sends it as `RP_VINFO` (fixed-size numbers, when they change), the bridge
turns it into `{"t":"vinfo"}` with numbers only.
+**Only what changed costs anything.** The compositor tells the mirror where it drew (`mirror_frame`'s
+damage box): only those 64×64 tiles are compared with the last frame (all of them twice a second
+anyway, in case), and each of the encoder's three input slots copies only what changed since it was
+last written. A session without a monitor reads the scene directly (no copy) unless a fullscreen app
+paints over it.
+
**The GPU keeps composing.** Until 0.8.3 a remote viewer switched the session to CPU composition.
Now `gpu_frame` reads the composed frame back (`gpu_read_next`) once, for the encoder and the tiles,
and the screen keeps its GPU composition. Only two cases still use the CPU path: protected windows on
diff --git a/server/src/main.c b/server/src/main.c
index 000be29..d0c9e77 100644
--- a/server/src/main.c
+++ b/server/src/main.c
@@ -1243,7 +1243,10 @@ static void present_cpu(Server *s, Out *o, int index, Rect r, int feed) {
const Canvas *seen = media_composed(&out, mirror_watchers() > 0, &mc);
view_reset();
mirror_composed_on_gpu(0); /* this frame was painted on the CPU */
- mirror_frame(seen->px, seen->stride, W, H);
+ /* only what went into this buffer can differ from the last frame (not with a fullscreen app
+ * painted over it, nor when the frame is the copy with protected windows in black) */
+ int dmg[4] = { need.x0, need.y0, need.x1, need.y1 };
+ mirror_frame(seen->px, seen->stride, W, H, seen == &out && !wl_fullscreen_active() ? dmg : NULL);
mirror_cursor(px_, py_, k.mode, k.px, k.w, k.h, k.hx, k.hy, k.gen);
}
cursor_local(s, &k, out_scale(o)); /* the arrow is also an image (cursor.c) */
@@ -1283,11 +1286,22 @@ static void present_cpu(Server *s, Out *o, int index, Rect r, int feed) {
}
/* Memory frames (headless session, the capture view): scene + fullscreen, then the remote desktop. */
-static void present_memory(Server *s, Out *o, int index, int feed) {
+static void present_memory(Server *s, Out *o, int index, int feed, Rect drawn) {
if (!o->frame) return;
+ /* Without a fullscreen app the frame is the scene itself: read it there, no copy. A fullscreen app
+ * paints over a copy (o->frame). Right after one, the whole screen counts as changed. */
+ static int fs_before;
+ int fs = wl_fullscreen_active(), whole = fs || fs_before;
+ fs_before = fs;
Canvas out;
- canvas_init(&out, o->frame, o->scene.w, o->scene.h, o->scene.w);
- canvas_copy(&out, &o->scene);
+ if (fs) {
+ canvas_init(&out, o->frame, o->scene.w, o->scene.h, o->scene.w);
+ canvas_copy(&out, &o->scene);
+ } else out = o->scene;
+ if (drawn.x0 < 0) drawn.x0 = 0;
+ if (drawn.y0 < 0) drawn.y0 = 0;
+ if (drawn.x1 > out.w) drawn.x1 = out.w;
+ if (drawn.y1 > out.h) drawn.y1 = out.h;
view_of(o, index);
wl_paint_fullscreen(&out);
Cur k;
@@ -1299,7 +1313,8 @@ static void present_memory(Server *s, Out *o, int index, int feed) {
MediaCursor mc = { x, y, k.mode, k.px, k.w, k.h, k.hx, k.hy };
const Canvas *seen = media_composed(&out, mirror_watchers() > 0, &mc);
mirror_composed_on_gpu(0); /* this frame was painted on the CPU */
- mirror_frame(seen->px, seen->stride, o->scene.w, o->scene.h);
+ int dmg[4] = { drawn.x0, drawn.y0, drawn.x1, drawn.y1 };
+ mirror_frame(seen->px, seen->stride, o->scene.w, o->scene.h, seen == &out && !whole ? dmg : NULL);
mirror_cursor(x, y, k.mode, k.px, k.w, k.h, k.hx, k.hy, k.gen);
}
view_reset();
@@ -1353,8 +1368,8 @@ static void render_frame(Server *s) {
int main_shown = 0; /* the main display got this frame */
if (s->headless) {
Out *o = &s->out[0];
- paint_out(s, o, 0, 1);
- present_memory(s, o, 0, 1);
+ Rect drawn = paint_out(s, o, 0, 1);
+ present_memory(s, o, 0, 1, drawn);
} else {
for (int i = 0; i < OUT_MAX; i++) {
Out *o = &s->out[i];
@@ -1387,7 +1402,7 @@ static void render_frame(Server *s) {
int cx, cy;
pointer_phys(s, o, &cx, &cy);
mirror_composed_on_gpu(1);
- mirror_frame(rb, o->scene.w, o->scene.w, o->scene.h);
+ mirror_frame(rb, o->scene.w, o->scene.w, o->scene.h, NULL);
mirror_cursor(cx, cy, k.mode, k.px, k.w, k.h, k.hx, k.hy, k.gen);
}
view_reset();
@@ -1413,8 +1428,8 @@ static void render_frame(Server *s) {
c->full = out_frames(c, 1) == 0;
}
if (c->scene.px && (c->full || !box_empty(c->pend) || c->cdirty || wl_fullscreen_active())) {
- paint_out(s, c, OUT_MAX, 1);
- present_memory(s, c, OUT_MAX, 1);
+ Rect drawn = paint_out(s, c, OUT_MAX, 1);
+ present_memory(s, c, OUT_MAX, 1, drawn);
}
} else if (c->on) { out_free(c); c->on = 0; c->pend = (Box){ 0, 0, 0, 0 }; }
}
diff --git a/server/src/media/mstream.c b/server/src/media/mstream.c
index 7264818..4286e38 100644
--- a/server/src/media/mstream.c
+++ b/server/src/media/mstream.c
@@ -33,6 +33,7 @@ typedef struct {
size_t len, cap, total;
long frames, enc100;
int hw;
+ int dirty[4][4]; /* per input slot: what changed since it was last written (x0, y0, x1, y1) */
} Enc;
static Enc E[MS_CODECS];
@@ -143,6 +144,7 @@ static int start(Enc *e, int codec) {
e->w = fw;
e->h = fh;
e->t0 = now_ms();
+ for (int i = 0; i < 4; i++) { e->dirty[i][0] = e->dirty[i][1] = 0; e->dirty[i][2] = fw; e->dirty[i][3] = fh; }
e->last_feed = 0;
e->pending = 1;
e->sent_kbps = e->kbps;
@@ -210,8 +212,13 @@ static void feed(Enc *e, int codec) {
int slot = -1;
for (int i = 0; i < s->in_slots; i++) if (!s->in_busy[i]) { slot = i; break; }
if (slot < 0) return; /* the encoder is behind: the newest frame waits */
+ /* only what changed since this slot was last written: the rest of it is already the screen */
uint32_t *dst = (uint32_t *)(s->in_map + (size_t)slot * e->w * e->h * 4);
- for (int y = 0; y < e->h; y++) memcpy(dst + (size_t)y * e->w, fpx + (size_t)y * fstride, (size_t)e->w * 4);
+ int *d = slot < 4 ? e->dirty[slot] : NULL, full[4] = { 0, 0, e->w, e->h };
+ if (!d) d = full;
+ int x0 = d[0] < 0 ? 0 : d[0], y0 = d[1] < 0 ? 0 : d[1], x1 = d[2] > e->w ? e->w : d[2], y1 = d[3] > e->h ? e->h : d[3];
+ for (int y = y0; y < y1 && x1 > x0; y++) memcpy(dst + (size_t)y * e->w + x0, fpx + (size_t)y * fstride + x0, (size_t)(x1 - x0) * 4);
+ if (slot < 4) d[0] = d[1] = d[2] = d[3] = 0;
if (e->kbps != e->sent_kbps) { mp_sendv(s->sock, MM_RATE, 0, e->kbps, 0, 0, 0); e->sent_kbps = e->kbps; }
if (e->want_key) { mp_sendv(s->sock, MM_KEYFRAME, 0, 0, 0, 0, 0); e->want_key = 0; }
if (mp_sendv(s->sock, MM_INFRAME, (uint32_t)slot, (int64_t)(t - e->t0), 0, 0, 0)) return;
@@ -222,15 +229,27 @@ static void feed(Enc *e, int codec) {
e->flushing = 0;
}
-void mstream_frame(const uint32_t *px, int stride, int w, int h) {
+void mstream_frame(const uint32_t *px, int stride, int w, int h, const int *box) {
fpx = px;
fstride = stride;
fw = w & ~1;
fh = h & ~1;
+ int all[4] = { 0, 0, fw, fh };
+ if (!box) box = all;
for (int c = 1; c < MS_CODECS; c++) {
Enc *e = &E[c];
if (!e->viewers) continue;
if (e->id && (e->w != fw || e->h != fh)) stop(e); /* new screen size: a new encoder */
+ for (int i = 0; i < 4; i++) { /* every slot misses this change */
+ int *d = e->dirty[i];
+ if (d[2] <= d[0] || d[3] <= d[1]) memcpy(d, box, sizeof all);
+ else {
+ if (box[0] < d[0]) d[0] = box[0];
+ if (box[1] < d[1]) d[1] = box[1];
+ if (box[2] > d[2]) d[2] = box[2];
+ if (box[3] > d[3]) d[3] = box[3];
+ }
+ }
e->pending = 1;
feed(e, c);
}
diff --git a/server/src/media/mstream.h b/server/src/media/mstream.h
index 8279abc..d9b4f8c 100644
--- a/server/src/media/mstream.h
+++ b/server/src/media/mstream.h
@@ -18,7 +18,8 @@ void mstream_init(const MStreamOut *out);
/* How many viewers want this codec, with sound, and the lowest bit rate among them. 0 viewers stops it. */
void mstream_want(int codec, int viewers, int audio, unsigned kbps, int fps);
/* The session's screen changed (the frame stays valid until the next call or mstream_want(…, 0)). */
-void mstream_frame(const uint32_t *px, int stride, int w, int h);
+/* box: what changed since the last call (x0, y0, x1, y1), or NULL for all of it. */
+void mstream_frame(const uint32_t *px, int stride, int w, int h, const int *box);
void mstream_keyframe(int codec);
void mstream_tick(void);
double mstream_timeout(void); /* ms until mstream_tick has work, or -1 */
diff --git a/server/src/remote/mirror.c b/server/src/remote/mirror.c
index 0ae0794..a3a4979 100644
--- a/server/src/remote/mirror.c
+++ b/server/src/remote/mirror.c
@@ -304,7 +304,7 @@ static void handle(Client *c, int type, const unsigned char *d, uint32_t n) {
c->vfps = hz >= 24 && hz <= 360 ? hz : 30;
if (c->vcodec && c->vcodec != was) c->vneed_key = 1;
video_update();
- if (c->vcodec && fb) mstream_frame(fb, W, W, H);
+ if (c->vcodec && fb) mstream_frame(fb, W, W, H, NULL);
else if (c->vcodec && cb.refresh) cb.refresh(); /* the first viewer: a whole frame */
}
break;
@@ -571,7 +571,10 @@ void mirror_dispatch(const struct pollfd *p, int n) {
}
}
-void mirror_frame(const uint32_t *px, int stride, int w, int h) {
+#define FULL_CHECK_MS 500 /* with damage hints, every tile is still compared this often */
+
+void mirror_frame(const uint32_t *px, int stride, int w, int h, const int *dmg) {
+ static double full_at;
if (lfd < 0 || !mirror_watchers()) { free(fb); fb = NULL; return; }
int resized = !fb || w != W || h != H, any = 0;
int new_size = w != W || h != H;
@@ -592,8 +595,19 @@ void mirror_frame(const uint32_t *px, int stride, int w, int h) {
} else {
if (!chg) chg = calloc((size_t)TW * TH, 1);
if (chg) memset(chg, 0, (size_t)TW * TH);
- for (int ty = 0; ty < TH; ty++)
- for (int tx = 0; tx < TW; tx++) {
+ /* only the tiles the compositor drew over (the whole screen now and then, in case) */
+ int tx0 = 0, ty0 = 0, tx1 = TW, ty1 = TH;
+ double t = now_ms();
+ if (dmg && t - full_at < FULL_CHECK_MS) {
+ tx0 = dmg[0] < 0 ? 0 : dmg[0] / TILE;
+ ty0 = dmg[1] < 0 ? 0 : dmg[1] / TILE;
+ tx1 = dmg[2] <= 0 ? 0 : (dmg[2] + TILE - 1) / TILE;
+ ty1 = dmg[3] <= 0 ? 0 : (dmg[3] + TILE - 1) / TILE;
+ if (tx1 > TW) tx1 = TW;
+ if (ty1 > TH) ty1 = TH;
+ } else full_at = t;
+ for (int ty = ty0; ty < ty1; ty++)
+ for (int tx = tx0; tx < tx1; tx++) {
int x = tx * TILE, y = ty * TILE;
int tw = x + TILE > W ? W - x : TILE, th = y + TILE > H ? H - y : TILE;
int changed = 0;
@@ -612,7 +626,20 @@ void mirror_frame(const uint32_t *px, int stride, int w, int h) {
send_damage();
int video = 0;
for (int i = 0; i < MAX_CLIENTS; i++) if (cl[i].fd >= 0 && cl[i].vcodec) video = 1;
- if (video) mstream_frame(fb, W, W, H);
+ if (video) {
+ /* the changed tiles' box: the encoder's slots copy only that */
+ int box[4] = { W, H, 0, 0 };
+ for (int ty = 0; chg && ty < TH; ty++)
+ for (int tx = 0; tx < TW; tx++) {
+ if (!chg[ty * TW + tx]) continue;
+ int x = tx * TILE, y = ty * TILE, x1 = x + TILE > W ? W : x + TILE, y1 = y + TILE > H ? H : y + TILE;
+ if (x < box[0]) box[0] = x;
+ if (y < box[1]) box[1] = y;
+ if (x1 > box[2]) box[2] = x1;
+ if (y1 > box[3]) box[3] = y1;
+ }
+ mstream_frame(fb, W, W, H, chg && box[2] > box[0] ? box : NULL);
+ }
}
for (int i = 0; i < MAX_CLIENTS; i++) pump(&cl[i]);
}
diff --git a/server/src/remote/mirror.h b/server/src/remote/mirror.h
index eec6031..42f1d82 100644
--- a/server/src/remote/mirror.h
+++ b/server/src/remote/mirror.h
@@ -50,8 +50,10 @@ void mirror_greeter(const char *json);
void mirror_shown(void);
/* A new screen frame (without the cursor), for the encoder; compares it tile by tile with the previous
- * one: the damage goes out at once, the lossless tiles once the screen is still. */
-void mirror_frame(const uint32_t *px, int stride, int w, int h);
+ * one: the damage goes out at once, the lossless tiles once the screen is still. dmg (x0, y0, x1, y1):
+ * where the compositor drew since the last call, so only those tiles are compared (all of them twice a
+ * second anyway); NULL: anywhere. */
+void mirror_frame(const uint32_t *px, int stride, int w, int h, const int *dmg);
/* Cursor: position and mode (0 arrow, 1 app image, 2 hidden); the image only if gen changed. */
void mirror_cursor(int x, int y, int mode, const uint32_t *argb, int w, int h, int hx, int hy, unsigned gen);
/* Lossless tiles for viewers whose video plays once the screen is still, the encoder's