Blender's GPU Accelerated UI, Decoupled from it's original codebase. Supports Rust, Python, JS, React, C & C++. Twice as good as GPUI
video decoder
Preetham Pemmasani74b99399cb61
diff --git a/CMakeLists.txt b/CMakeLists.txtindex e3d9cb4..278d6bc 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -99,6 +99,15 @@ if(BUI_BUILD_TESTS)BUI_TEST_TMP="${CMAKE_CURRENT_BINARY_DIR}/image_test_files")add_test(NAME bui_image_test COMMAND bui_image_test)set_tests_properties(bui_image_test PROPERTIES TIMEOUT 120)+ # video: play, pause, seek, rate, loop, off screen, malformed files, no leaks+ # (exits 77, skipped, without a backend that decodes the H.264 clip)+ add_executable(bui_video_test tests/video_test.c)+ target_link_libraries(bui_video_test PRIVATE bui::kit)+ target_compile_definitions(bui_video_test PRIVATE+ BUI_TEST_VIDEOS="${CMAKE_CURRENT_SOURCE_DIR}/tests/videos"+ BUI_TEST_TMP="${CMAKE_CURRENT_BINARY_DIR}")+ add_test(NAME bui_video_test COMMAND bui_video_test)+ set_tests_properties(bui_video_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 180)if(WIN32) # the thumbnail-grid benchmark (bench/images/image_grid.c; not a test)add_executable(bui_image_bench bench/images/image_grid.c)target_link_libraries(bui_image_bench PRIVATE bui::kit psapi)diff --git a/gpu/include/GPU_raster.h b/gpu/include/GPU_raster.hindex 289c8a7..bf310c4 100644--- a/gpu/include/GPU_raster.h+++ b/gpu/include/GPU_raster.h@@ -146,6 +146,9 @@ enum {enum {BUI_IMAGE_NEAREST = 1 << 0, /* nearest texel (CSS image-rendering: pixelated); else bilinear */};+/* Rows y0 .. y0 + rows of `img` as premultiplied 0xAARRGGBB (`dst`: w per+ * row): a copy, or NV12 converted the way drawing converts it. */+void BUI_pixmap_read(const BuiPixmap *img, int y0, int rows, uint32_t *dst);/* Draws the `src` part of `img` (x, y, w, h in image pixels; NULL = all of* it) into x, y, w, h, filtered, times `alpha`, premultiplied source-over.* 1:1 on whole pixels is an exact copy. */diff --git a/gpu/intern/gpu_raster.c b/gpu/intern/gpu_raster.cindex fb170a1..1a91077 100644--- a/gpu/intern/gpu_raster.c+++ b/gpu/intern/gpu_raster.c@@ -1300,6 +1300,23 @@ static inline uint32_t yuv_argb(const int *k, int y, int u, int v)return 0xFF000000u | (uint32_t)r << 16 | (uint32_t)g << 8 | (uint32_t)b;}+void BUI_pixmap_read(const BuiPixmap *img, int y0, int rows, uint32_t *dst)+{+ if (img->format == BUI_PIXMAP_ARGB) {+ memcpy(dst, img->px + (size_t)y0 * img->w, sizeof(uint32_t) * (size_t)img->w * rows);+ return;+ }+ const int *k = k_yuv[img->format == BUI_PIXMAP_NV12_BT709];+ const uint8_t *yp = (const uint8_t *)img->px, *uvp = yp + (size_t)img->w * img->h;+ for (int y = y0; y < y0 + rows; y++) {+ const uint8_t *yr = yp + (size_t)y * img->w, *cr = uvp + (size_t)(y >> 1) * img->w;+ for (int x = 0; x < img->w; x++) {+ const uint8_t *c = cr + (x & ~1);+ *dst++ = yuv_argb(k, yr[x], c[0], c[1]);+ }+ }+}+/* 8-bit bilinear of four samples (fx, fy: 0..256). */static inline int lerp2(int a, int b, int c, int d, int fx, int fy){diff --git a/kit/CMakeLists.txt b/kit/CMakeLists.txtindex d764dc0..11e8e46 100644--- a/kit/CMakeLists.txt+++ b/kit/CMakeLists.txt@@ -44,6 +44,9 @@ add_library(bui_kit STATICintern/image/cache.cintern/image/loader.cintern/image/draw.c+ intern/image/video.c+ intern/image/movie.c+ intern/image/movie_mf.c# Blender's UI sources copied and edited for the kit (C++; see blender/bk_blender.h)intern/blender/interface_string_search.ccintern/blender/interface_utils.cc@@ -62,6 +65,9 @@ option(BUI_IMAGE_SVG "SVG images through the kit's nanosvg (no extra code: the iif(BUI_IMAGE_SVG)target_compile_definitions(bui_kit PRIVATE BUI_IMAGE_SVG)endif()+if(WIN32)+ target_compile_definitions(bui_kit PRIVATE BUI_VIDEO_MF)+endif()set(_bui_quiet "$<IF:$<C_COMPILER_ID:MSVC>,/w,-w>")# Wuffs: one translation unit (format_wuffs.c includes wuffs-v0.4.c)target_include_directories(bui_kit PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/extern/wuffs)diff --git a/kit/include/BUI_kit.h b/kit/include/BUI_kit.hindex a374cc3..9b3437b 100644--- a/kit/include/BUI_kit.h+++ b/kit/include/BUI_kit.h@@ -260,6 +260,7 @@ enum {BK_IMAGE_HDR,BK_IMAGE_PNM,BK_IMAGE_SVG,+ BK_IMAGE_VIDEO, /* a frame of a BKVideo */};/* CSS object-fit */enum { BK_FIT_FILL = 0, BK_FIT_CONTAIN, BK_FIT_COVER, BK_FIT_NONE, BK_FIT_SCALE_DOWN };@@ -303,6 +304,48 @@ BKImBuf *BK_anim_next(BKAnim *anim, int *r_delay_ms);int BK_anim_info(const BKAnim *anim, int *r_w, int *r_h, int *r_loops);void BK_anim_free(BKAnim *anim);+/* -- video (kit/intern/image: one player for GIF, WebP and video) ---------------------+ * A <video>: decoded by the OS (Media Foundation, AVFoundation, GStreamer) or+ * an optional built-in decoder (dav1d, FFmpeg), streamed from the file at+ * the size it is drawn, frames in one GPU texture updated in place (NV12:+ * YCbCr converted in the shader; the CPU raster converts what it draws).+ * It loads when first drawn and plays only while it shows: off screen,+ * clipped, in a hidden window, nothing is decoded. Muted (no audio yet).+ * UI-thread API; the state listener runs on the UI thread. */+typedef struct BKVideo BKVideo;+enum { BK_VIDEO_LOADING = 0, BK_VIDEO_PLAYING, BK_VIDEO_PAUSED, BK_VIDEO_ENDED, BK_VIDEO_ERROR };+/* Autoplays (a muted autoplay, as browsers allow); BK_video_pause before the+ * first draw shows the first frame instead. NULL for an empty src. */+BKVideo *BK_video_new(const char *src);+void BK_video_free(BKVideo *v);+void BK_video_play(BKVideo *v); /* an ended video starts over */+void BK_video_pause(BKVideo *v);+void BK_video_seek(BKVideo *v, double seconds); /* the frame there shows, paused or not */+void BK_video_set_rate(BKVideo *v, double rate); /* playback speed, 1/16 .. 16 */+void BK_video_set_loop(BKVideo *v, int loop);+/* An image file shown until the first frame (HTML poster); NULL: none. */+void BK_video_set_poster(BKVideo *v, const char *src);+/* Blender-style controls over it (play/pause, scrubber, time) in "video" nodes. */+void BK_video_set_controls(BKVideo *v, int on);+int BK_video_state(const BKVideo *v); /* BK_VIDEO_* */+double BK_video_time(const BKVideo *v); /* seconds */+double BK_video_duration(const BKVideo *v); /* seconds; 0 until loaded or unknown */+int BK_video_size(const BKVideo *v, int *r_w, int *r_h); /* intrinsic; 0 until loaded */+/* Called on every state change (loaded, playing, paused, ended, error). */+void BK_video_set_listener(BKVideo *v, void (*fn)(BKVideo *v, int state, void *data), void *data);+/* Dear ImGui and textured meshes: the current frame in a BUI_texture (RGBA),+ * created when `tex` is 0 (or the size changed: the old one is freed) and+ * updated in place when a new frame came; returns the texture (0 before+ * the first frame). Calling it each frame is what keeps it playing: the+ * host keeps rendering while BK_video_state is BK_VIDEO_PLAYING. */+int BK_video_texture(BKVideo *v, int tex, int max_w, int max_h);+/* Shown on a "video" node (fitted, with its controls) or an "image" node; the+ * node borrows it (free the video after the node lets go: set NULL). */+void BK_node_set_video(BKNode *node, BKVideo *v, int fit);+/* The backends this build reads video with ("mf", "avfoundation", "gstreamer",+ * "dav1d", "ffmpeg"), comma-separated; "" when built without video. */+const char *BK_video_backends(void);+/* The shared cache and loader. Sources (file paths) decode on a few worker* threads at the size they are drawn, visible first; results arrive on the UI* thread (the window loop wakes for them) and redraw only where they show. */@@ -355,6 +398,7 @@ typedef struct BKCanvasItem {int fit; /* BK_FIT_* */int frame; /* animated src: 0 plays (each frame for its delay, looping as the file* says; paused while off screen); n > 0 shows frame n - 1 */+ BKVideo *video; /* images: a video's current frame (borrowed, as imbuf) */} BKCanvasItem;void BK_canvas_set_item(BKNode *canvas, const BKCanvasItem *item); /* insert or replace by id */void BK_canvas_add_item(BKNode *canvas, const BKCanvasItem *item);diff --git a/kit/intern/image/anim.c b/kit/intern/image/anim.cindex 34922cf..37302db 100644--- a/kit/intern/image/anim.c+++ b/kit/intern/image/anim.c@@ -1,12 +1,15 @@-/* Animations (GIF, animated WebP) decoded one frame at a time, at the size- * they are drawn: the cache keeps the frame on screen and the next one,- * not the whole animation. Each format provides a BKImAnimCodec that- * composites its frames (GIF disposal and blending, WebP's canvas) into a- * full-size premultiplied frame; this file fits frames to the target (the- * same plan as stills, imb_plan / imb_finish) and loops.+/* Animated media, one frame at a time, at the size drawn: GIF, animated+ * WebP and video behind one codec interface (BKImAnimCodec: Blender's+ * MovieReader shape, open / next / seek / close). In-memory codecs composite+ * frames (GIF disposal and blending, WebP's canvas) into a full-size+ * premultiplied canvas, fitted here with the plan of stills (imb_plan /+ * imb_finish); streamed codecs (movie.c: the OS decoders, dav1d, FFmpeg)+ * read the file as they go and decode NV12 at about the size drawn into a+ * recycled frame (no allocation per frame).** SPDX-License-Identifier: GPL-2.0-or-later */+#include <math.h>#include <string.h>#include "MEM_guardedalloc.h"@@ -16,12 +19,17 @@struct BKImAnim {const BKImAnimCodec *codec;void *state;- uint8_t *file; /* owned copy of the encoded file (the codecs read from it) */+ uint8_t *file; /* in-memory codecs: the encoded file they read (owned) */size_t size;- int w, h; /* canvas (intrinsic) size */- int frames, loops;+ char *path; /* streamed codecs: the file (owned) */+ BKImAnimInfo info;+ int frames; /* from the header, at least what was decoded */int ftype; /* BK_IMAGE_* */int index; /* the frame next() returns next */+ double pts; /* its time (seconds) when the codec doesn't say */+ double seek; /* the frame to return next shows at this time (< 0: none) */+ uint64_t stream; /* streamed: the frames' BuiPixmap stream (one GPU texture) */+ BKImageFit target;BKImPlan plan; /* how frames are fitted */};@@ -36,31 +44,97 @@ const BKImAnimCodec *imb_anim_codec(const uint8_t *mem, size_t size)return NULL;}+static BKImAnim *anim_new(const BKImAnimCodec *codec, const BKImageFit *target)+{+ BKImAnim *a = (BKImAnim *)MEM_callocN(sizeof(BKImAnim), "BKImAnim");+ a->codec = codec;+ a->target = *target;+ a->seek = -1.0;+ return a;+}+BKImAnim *imb_anim_open(uint8_t *file, size_t size, const BKImageFit *target){BKImageInfo info;const BKImFileType *t = imb_file_type(file, size);const BKImAnimCodec *codec = NULL;- BKImAnim *a = NULL;if (t && t->load_header(file, size, &info) && info.width > 0 && info.height > 0) {codec = imb_anim_codec(file, size);}- if (codec) {- a = (BKImAnim *)MEM_callocN(sizeof(BKImAnim), "BKImAnim");- a->codec = codec;- a->ftype = t->ftype;- a->file = file;- a->size = size;- a->state = codec->open(file, size, &a->w, &a->h, &a->loops);- a->frames = BK_MAX(1, info.frames);- if (!a->state || !imb_plan(target, a->w, a->h, 1, &a->plan)) {- imb_anim_free(a); /* frees `file` too */- return NULL;+ if (!codec) {+ MEM_freeN(file);+ return NULL;+ }+ BKImAnim *a = anim_new(codec, target);+ a->ftype = t->ftype;+ a->file = file;+ a->size = size;+ a->frames = BK_MAX(1, info.frames);+ a->state = codec->open(NULL, file, size, 0, 0, &a->info);+ if (!a->state || !imb_plan(target, a->info.w, a->info.h, 1, &a->plan)) {+ imb_anim_free(a); /* frees `file` too */+ return NULL;+ }+ return a;+}++/* Opens `a`'s streamed codec at its target (the plan's whole-frame size). */+static bool movie_start(BKImAnim *a)+{+ BKImPlan plan;+ if (a->info.src_w > 0 && !imb_plan(&a->target, a->info.src_w, a->info.src_h, 1, &plan)) {+ return false; /* nothing of it shows */+ }+ const int want_w = a->info.src_w > 0 ? plan.fw : a->target.box_w;+ const int want_h = a->info.src_h > 0 ? plan.fh : a->target.box_h;+ BKImAnimInfo info;+ memset(&info, 0, sizeof(info));+ void *state = a->codec->open(a->path, NULL, 0, want_w, want_h, &info);+ if (!state) {+ return false;+ }+ if (info.w <= 0 || info.h <= 0 || (info.w | info.h) & 1 || info.w > BK_IMB_MAX_SIDE ||+ info.h > BK_IMB_MAX_SIDE || info.src_w <= 0 || info.src_h <= 0 ||+ !imb_plan(&a->target, info.src_w, info.src_h, 1, &plan))+ {+ a->codec->close(state); /* validated at the codec boundary: frames are even-sized NV12 */+ return false;+ }+ if (a->state) {+ a->codec->close(a->state);+ }+ a->state = state;+ a->info = info;+ a->plan = plan;+ return true;+}++BKImAnim *imb_movie_open(const char *path, const BKImageFit *target)+{+ size_t n;+ uint8_t *head = imb_read_file(path, 64, &n);+ if (!head) {+ return NULL;+ }+ BKImAnim *a = NULL;+ for (int i = 0; imb_movie_codecs[i] && !a; i++) {+ const BKImAnimCodec *codec = imb_movie_codecs[i];+ if (codec->is_a && !codec->is_a(head, n)) {+ continue;+ }+ a = anim_new(codec, target);+ const size_t len = strlen(path);+ a->path = (char *)MEM_mallocN(len + 1, "movie path");+ memcpy(a->path, path, len + 1);+ a->ftype = BK_IMAGE_VIDEO;+ a->stream = imb_new_key();+ if (!movie_start(a)) {+ imb_anim_free(a);+ a = NULL; /* the next backend may read it */}- return a;}- MEM_freeN(file);- return NULL;+ MEM_freeN(head);+ return a;}void imb_anim_free(BKImAnim *a)@@ -69,14 +143,18 @@ void imb_anim_free(BKImAnim *a)if (a->state) {a->codec->close(a->state);}+ if (a->stream) {+ BUI_image_forget(a->stream); /* its GPU texture */+ }MEM_freeN(a->file);+ MEM_freeN(a->path);MEM_freeN(a);}}-int imb_anim_loops(const BKImAnim *a)+const BKImAnimInfo *imb_anim_info(const BKImAnim *a){- return a->loops;+ return &a->info;}int imb_anim_frames(const BKImAnim *a)@@ -89,39 +167,124 @@ size_t imb_anim_bytes(const BKImAnim *a)return sizeof(*a) + a->size + a->codec->bytes(a->state);}-/* The next composited frame at full size (a fresh buffer), wrapping to the- * first after the last. NULL on a decoding error. */-static BKImBuf *next_full(BKImAnim *a, int *r_delay, bool *r_wrapped)+bool imb_anim_retarget(BKImAnim *a, const BKImageFit *target){- BKImBuf *ib = imb_alloc(a->w, a->h, 1);- if (!ib) {- return NULL;+ if (memcmp(&a->target, target, sizeof(*target)) == 0) {+ return true;}- ib->ftype = a->ftype;+ const BKImageFit old = a->target;+ a->target = *target;+ if (!a->codec->streamed) {+ BKImPlan plan;+ if (imb_plan(target, a->info.w, a->info.h, 1, &plan)) {+ a->plan = plan;+ return true;+ }+ }+ else if (movie_start(a)) {+ imb_anim_seek(a, a->pts); /* where it was */+ return true;+ }+ a->target = old;+ return false;+}++void imb_anim_seek(BKImAnim *a, double t)+{+ a->seek = BK_MAX(0.0, t);+}++/* The next frame from the codec into `dst`, wrapping to the first after the+ * last; -1 on errors. */+static int codec_next(BKImAnim *a, BKImBuf *dst, double *r_pts, double *r_dur, bool *r_wrapped)+{*r_wrapped = false;- int got = a->codec->next(a->state, ib->rect, r_delay);- if (got == 0 && a->index > 0) { /* past the last frame: from the top */- a->codec->rewind(a->state);+ if (a->seek >= 0.0) {+ a->codec->seek(a->state, a->seek);a->index = 0;- *r_wrapped = true;- got = a->codec->next(a->state, ib->rect, r_delay);+ a->pts = 0.0;}- if (got <= 0) {- MEM_freeN(ib);+ for (;;) {+ double pts = -1.0, dur = 0.0;+ int got = a->codec->next(a->state, dst, &pts, &dur);+ if (got == 0 && a->index > 0) { /* past the last frame: from the top */+ a->codec->seek(a->state, 0.0);+ a->index = 0;+ a->pts = 0.0;+ *r_wrapped = true;+ got = a->codec->next(a->state, dst, &pts, &dur);+ }+ if (got <= 0) {+ a->seek = -1.0;+ return -1;+ }+ if (!a->codec->streamed && dur <= 0.0105) {+ dur = 0.1; /* GIF/WebP 0 and 10 ms mean "as fast as possible": browsers show 100 ms */+ }+ pts = pts >= 0.0 ? pts : a->pts;+ a->pts = pts + dur;+ a->index++;+ a->frames = BK_MAX(a->frames, a->index); /* headers can undercount */+ /* a seek lands on the frame showing at its time: skip the ones before */+ if (a->seek >= 0.0 && pts + dur <= a->seek + 1e-6 && !*r_wrapped) {+ continue;+ }+ a->seek = -1.0;+ *r_pts = pts;+ *r_dur = dur;+ return 1;+ }+}++/* The next composited frame at full size (a fresh buffer), in-memory codecs. */+static BKImBuf *next_full(BKImAnim *a, double *r_pts, double *r_dur, bool *r_wrapped)+{+ BKImBuf *ib = imb_alloc(a->info.w, a->info.h, 1);+ if (!ib) {return NULL;}- a->index++;- a->frames = BK_MAX(a->frames, a->index); /* headers can undercount */- if (*r_delay <= 10) {- *r_delay = 100; /* 0 and 10 ms mean "as fast as possible": browsers show 100 ms */+ ib->ftype = a->ftype;+ if (codec_next(a, ib, r_pts, r_dur, r_wrapped) < 0) {+ MEM_freeN(ib);+ return NULL;}return ib;}-BKImBuf *imb_anim_next(BKImAnim *a, int *r_delay, bool *r_wrapped)+BKImBuf *imb_anim_next(BKImAnim *a, BKImBuf *reuse, double *r_pts, double *r_dur, bool *r_wrapped){- BKImBuf *ib = next_full(a, r_delay, r_wrapped);- return ib ? imb_finish(ib, &a->plan) : NULL;+ if (!a->codec->streamed) {+ BK_imbuf_free(reuse); /* composited and fitted into fresh buffers */+ BKImBuf *ib = next_full(a, r_pts, r_dur, r_wrapped);+ return ib ? imb_finish(ib, &a->plan) : NULL;+ }+ BKImBuf *ib = reuse;+ if (ib && (ib->x != a->info.w || ib->y != a->info.h || ib->format != a->info.format)) {+ BK_imbuf_free(ib); /* resized */+ ib = NULL;+ }+ if (!ib) {+ ib = imb_alloc_nv12(a->info.w, a->info.h, a->info.format);+ if (!ib) {+ return NULL;+ }+ ib->ftype = BK_IMAGE_VIDEO;+ ib->stream = a->stream;+ }+ if (codec_next(a, ib, r_pts, r_dur, r_wrapped) < 0) {+ BK_imbuf_free(ib);+ return NULL;+ }+ imb_rekey(ib); /* new pixels: GPU renderers update the stream's texture */+ /* the whole frame is decoded; the visible part is drawn from it */+ const double kx = ib->x / (double)a->info.src_w, ky = ib->y / (double)a->info.src_h;+ ib->src_x = a->info.src_w;+ ib->src_y = a->info.src_h;+ ib->crop[0] = (float)(a->plan.src[0] * kx);+ ib->crop[1] = (float)(a->plan.src[1] * ky);+ ib->crop[2] = (float)(a->plan.src[2] * kx);+ ib->crop[3] = (float)(a->plan.src[3] * ky);+ return ib;}BKImBuf *imb_anim_frame(const uint8_t *mem, size_t size, const BKImageFit *target, int frame)@@ -134,10 +297,10 @@ BKImBuf *imb_anim_frame(const uint8_t *mem, size_t size, const BKImageFit *targeBKImAnim *a = imb_anim_open(copy, size, target);BKImBuf *ib = NULL;for (int i = 0; a && i <= frame; i++) {- int delay;+ double pts, dur;bool wrapped;MEM_freeN(ib);- ib = next_full(a, &delay, &wrapped);+ ib = next_full(a, &pts, &dur, &wrapped);if (!ib || wrapped) { /* past the end: no such frame */MEM_freeN(ib);ib = NULL;@@ -163,21 +326,22 @@ BKImBuf *imb_anim_all(const uint8_t *mem, size_t size, const BKImageFit *target)return NULL;}/* every frame in one buffer (the synchronous API), at most 128 MiB of them */+ const int w = a->info.w, h = a->info.h;const int frames = (int)BK_MIN((size_t)a->frames,- BK_MAX((size_t)1, ((size_t)1 << 25) / ((size_t)a->w * a->h)));- BKImBuf *all = imb_alloc(a->w, a->h, frames);+ BK_MAX((size_t)1, ((size_t)1 << 25) / ((size_t)w * h)));+ BKImBuf *all = imb_alloc(w, h, frames);int n = 0;while (all && n < frames) {- int delay;+ double pts, dur;bool wrapped;- BKImBuf *ib = next_full(a, &delay, &wrapped);+ BKImBuf *ib = next_full(a, &pts, &dur, &wrapped);if (!ib || wrapped) {MEM_freeN(ib);break;}- memcpy(imb_frame(all, n), ib->rect, sizeof(uint32_t) * (size_t)a->w * a->h);+ memcpy(imb_frame(all, n), ib->rect, sizeof(uint32_t) * (size_t)w * h);if (all->delays) {- all->delays[n] = delay;+ all->delays[n] = (int)lround(dur * 1000.0);}MEM_freeN(ib);n++;@@ -188,7 +352,7 @@ BKImBuf *imb_anim_all(const uint8_t *mem, size_t size, const BKImageFit *target)}if (all) {all->frames = n;- all->loops = a->loops;+ all->loops = a->info.loops;all->ftype = a->ftype;all = imb_finish(all, &a->plan);}@@ -213,11 +377,11 @@ BKAnim *BK_anim_open(const void *data, size_t len, int max_w, int max_h)BKImBuf *BK_anim_next(BKAnim *anim, int *r_delay_ms){- int delay = 0;+ double pts = 0.0, dur = 0.0;bool wrapped;- BKImBuf *ib = anim ? imb_anim_next((BKImAnim *)anim, &delay, &wrapped) : NULL;+ BKImBuf *ib = anim ? imb_anim_next((BKImAnim *)anim, NULL, &pts, &dur, &wrapped) : NULL;if (r_delay_ms) {- *r_delay_ms = delay;+ *r_delay_ms = (int)lround(dur * 1000.0);}return ib;}@@ -226,13 +390,13 @@ int BK_anim_info(const BKAnim *anim, int *r_w, int *r_h, int *r_loops){const BKImAnim *a = (const BKImAnim *)anim;if (r_w) {- *r_w = a ? a->w : 0;+ *r_w = a ? a->info.w : 0;}if (r_h) {- *r_h = a ? a->h : 0;+ *r_h = a ? a->info.h : 0;}if (r_loops) {- *r_loops = a ? a->loops : 0;+ *r_loops = a ? a->info.loops : 0;}return a ? a->frames : 0;}diff --git a/kit/intern/image/cache.c b/kit/intern/image/cache.cindex da0fcf1..3ebfed2 100644--- a/kit/intern/image/cache.c+++ b/kit/intern/image/cache.c@@ -10,6 +10,9 @@* two generations (they are on screen): visible images can exceed the* budget, the rest can't.* - Deterministic: BK_image_exit frees everything; MEM counts all of it.+ * - Private entries (a <video> each, video.c) use the same requests,+ * clock and visibility tracking, outside the table and the LRU: their+ * owner frees them.** SPDX-License-Identifier: GPL-2.0-or-later */@@ -31,23 +34,32 @@ typedef struct Waiter {unsigned pass; /* the drawing pass that last asked */} Waiter;-typedef struct Entry {- struct Entry *lru_prev, *lru_next; /* most recently used first */- struct Entry *pend_prev, *pend_next; /* while a request is out */+typedef struct BKImEntry Entry;+struct BKImEntry {+ Entry *lru_prev, *lru_next; /* most recently used first */+ Entry *pend_prev, *pend_next; /* while a request is out */BKImageSource *source; /* referenced */BKImageFit target;BKImBuf *ib; /* NULL while pending or after a failed decode */BKImRequest *req; /* the request out (NULL when settled) */+ BKImRequest *spare; /* a step's request, kept for the next step (no allocation per frame) */unsigned used; /* generation of the last use */size_t bytes; /* counted in C.bytes: ib, and play's decoder and next frame */BKImPlay *play; /* a playing animation (NULL for stills) */- struct Entry *play_prev, *play_next; /* C.playing */+ Entry *play_prev, *play_next; /* C.playing */+ /* private entries (owned: video.c): who hears state changes, and the+ * last state it heard; orphaned when the owner let go with a request out */+ bool owned, orphan;+ unsigned hosted; /* C.gen + 1 when a host loop last showed it without a rect (textures) */+ void (*notify)(void *owner);+ void *owner;+ BKImPlayState heard;/* where it shows (window rects per context): while pending, what redraws* on arrival; for animations, what redraws on every frame (a context* that redraws a rect without asking drops it: off screen, paused) */int nwaiters;Waiter waiters[MAX_WAITERS];-} Entry;+};struct BKImageSource {int refs;@@ -190,8 +202,10 @@ static void entry_account(Entry *e)static void entry_free(Entry *e){- BLI_ghash_remove(C.entries, e, NULL, NULL);- lru_unlink(e);+ if (!e->owned) {+ BLI_ghash_remove(C.entries, e, NULL, NULL);+ lru_unlink(e);+ }if (e->play) {*(e->play_prev ? &e->play_prev->play_next : &C.playing) = e->play_next;if (e->play_next) {@@ -202,15 +216,29 @@ static void entry_free(Entry *e)C.bytes -= e->bytes;BK_imbuf_free(e->ib);bk_image_source_release(e->source);+ MEM_freeN(e->spare);MEM_freeN(e);C.count--;}-/* `anim`: a step of a playing animation (its next frame), else a first load. */-static void request(Entry *e, int priority, BKImAnim *anim)+/* A first load, or (`step`) the next frame of what plays when it has one to decode. */+static void request(Entry *e, int priority, bool step){- BKImRequest *req = (BKImRequest *)MEM_callocN(sizeof(BKImRequest), "BKImRequest");+ BKImBuf *reuse = NULL;+ double seek = -1.0;+ BKImAnim *anim = step ? imb_play_take_step(e->play, &reuse, &seek) : NULL;+ if (step && !anim) {+ return; /* nothing to decode now */+ }+ BKImRequest *req = e->spare && step ? e->spare :+ (BKImRequest *)MEM_mallocN(sizeof(BKImRequest), "BKImRequest");+ e->spare = req == e->spare ? NULL : e->spare;+ memset(req, 0, sizeof(*req));+ req->step = step;req->anim = anim;+ req->reuse = reuse;+ req->seek = seek;+ req->movie = e->owned && !step;req->source = e->source;e->source->refs++; /* the worker reads its path */req->target = e->target;@@ -276,7 +304,7 @@ static Entry *lookup(BKImageSource *s, const BKImageFit *target, bool create, inBLI_ghash_insert(C.entries, e, e);lru_front(e);C.count++;- request(e, priority, NULL);+ request(e, priority, false);}else if (e) {if (e != C.lru_head) {@@ -337,18 +365,38 @@ static void arm(const Entry *e)}}+/* A private entry's owner hears every state change (loaded, ended, error). */+static void heard(Entry *e)+{+ const BKImPlayState st = e->play ? imb_play_state(e->play) : BK_PLAY_LOADING;+ if (e->notify && st != e->heard) {+ e->heard = st;+ e->notify(e->owner);+ }+}++static void playing_link(Entry *e)+{+ e->play_next = C.playing;+ if (C.playing) {+ C.playing->play_prev = e;+ }+ C.playing = e;+}+/* A step of a playing animation came back. */static void step_done(Entry *e, BKImRequest *req){if (req->state == BK_IMREQ_CANCELLED) {- imb_play_step_cancelled(e->play, req->anim);+ imb_play_step_cancelled(e->play, req->anim, req->reuse);}else {- imb_play_step_done(e->play, req->anim, req->result, req->delay, req->wrapped);+ imb_play_step_done(e->play, req->anim, req->result, req->pts, req->dur, req->wrapped);req->result = NULL;C.stats.decoded++;}req->anim = NULL;+ req->reuse = NULL;entry_account(e);arm(e);}@@ -360,34 +408,45 @@ int BK_image_poll(void)while (req) {BKImRequest *next = req->next;Entry *e = (Entry *)req->entry;- if (e && req->state == BK_IMREQ_CANCELLED && e->nwaiters > 0) {+ Entry *keeper = NULL; /* keeps a step's request for its next step */+ if (e && !e->orphan && req->state == BK_IMREQ_CANCELLED && e->nwaiters > 0 && !req->step) {bk_loader_push(req); /* asked for again after the cancel: still wanted */req = next;continue;}- if (e && e->play) {+ if (e && e->orphan) { /* its owner let go while this was out */+ e->req = NULL;+ pend_unlink(e);+ entry_free(e);+ }+ else if (e && req->step) {e->req = NULL;pend_unlink(e);step_done(e, req);+ keeper = e->spare ? NULL : e;}else if (e) {e->req = NULL;pend_unlink(e);if (req->state == BK_IMREQ_CANCELLED) {- entry_free(e);+ if (!e->owned) {+ entry_free(e);+ }C.stats.cancelled++;}else {e->ib = req->result;req->result = NULL;if (e->ib && req->anim) { /* it plays: keep the decoder, show where it is */- e->play = imb_play_new(req->anim, req->delay);- req->anim = NULL;- e->play_next = C.playing;- if (C.playing) {- C.playing->play_prev = e;+ if (!e->play) {+ e->play = imb_play_new(false, -1);+ playing_link(e);}- C.playing = e;+ imb_play_start(e->play, req->anim, req->pts, req->dur);+ req->anim = NULL;+ }+ else if (e->play) {+ imb_play_fail(e->play); /* a video that didn't open */}entry_account(e);if (e->ib) {@@ -404,13 +463,20 @@ int BK_image_poll(void)if (!e->play) {e->nwaiters = 0; /* a still: nothing more to redraw */}+ heard(e);n++;}}BK_imbuf_free(req->result);+ BK_imbuf_free(req->reuse);imb_anim_free(req->anim); /* a first load whose entry is gone */bk_image_source_release(req->source);- MEM_freeN(req);+ if (keeper) {+ keeper->spare = req;+ }+ else {+ MEM_freeN(req);+ }req = next;}return n;@@ -423,23 +489,21 @@ void bk_image_tick(void)}const double now = bk_time() + 0.002; /* timers fire a little early */for (Entry *e = C.playing; e; e = e->play_next) {- if (e->nwaiters == 0) {+ if (e->nwaiters == 0 && !(e->hosted && C.gen + 1 - e->hosted <= 1)) {continue; /* nobody shows it: paused, no decoding */}BKImBuf *frame = imb_play_advance(e->play, now);if (frame) {- BK_imbuf_free(e->ib);+ imb_play_recycle(e->play, e->ib); /* a movie decodes its next frame into it */e->ib = frame;entry_account(e);damage(e); /* only its rectangles redraw */}if (!e->req) {- BKImAnim *a = imb_play_take_step(e->play);- if (a) {- request(e, BK_IMAGE_PRIORITY_VISIBLE, a); /* the one after, while this shows */- }+ request(e, BK_IMAGE_PRIORITY_VISIBLE, true); /* the one after, while this shows */}arm(e);+ heard(e);}}@@ -562,6 +626,91 @@ void bk_image_forget_context(BKContext *ctx)/** \} */+/* -------------------------------------------------------------------- */+/** \name Private entries (video.c)+ * \{ */++BKImEntry *bk_image_entry_new(BKImageSource *s, BKImPlay *play, void (*notify)(void *owner),+ void *owner)+{+ Entry *e = (Entry *)MEM_callocN(sizeof(Entry), "image private entry");+ e->owned = true;+ e->source = s;+ s->refs++;+ e->play = play;+ e->notify = notify;+ e->owner = owner;+ playing_link(e);+ C.count++;+ entry_account(e);+ return e;+}++void bk_image_entry_free(BKImEntry *e)+{+ if (!e) {+ return;+ }+ if (e->req) {+ e->orphan = true; /* freed when the request comes back (BK_image_poll) */+ e->notify = NULL;+ e->nwaiters = 0;+ bk_loader_cancel(e->req);+ return;+ }+ entry_free(e);+}++/* Decoding at about the size drawn: a box much bigger, or much smaller, than+ * what the decoder makes asks for another size (resizing within that is+ * scaled when drawn, so a window resize doesn't reopen the decoder). */+static bool retarget_wanted(const BKImageFit *had, const BKImageFit *want)+{+ return had->fit != want->fit || want->box_w * 4 > had->box_w * 5 ||+ want->box_h * 4 > had->box_h * 5 || want->box_w * 2 < had->box_w ||+ want->box_h * 2 < had->box_h;+}++const BKImBuf *bk_image_entry_get(BKImEntry *e, const BKImageFit *target, const int rect[4])+{+ const bool loaded = imb_play_state(e->play) != BK_PLAY_LOADING;+ if ((!loaded && !e->req && imb_play_state(e->play) != BK_PLAY_ERROR) ||+ retarget_wanted(&e->target, target))+ {+ e->target = *target; /* the first load, or the next step, decodes at this size */+ }+ if (!loaded && !e->req && imb_play_state(e->play) != BK_PLAY_ERROR) {+ request(e, BK_IMAGE_PRIORITY_VISIBLE, false);+ }+ if (rect) {+ add_waiter(e, rect);+ }+ else {+ e->hosted = C.gen + 1;+ }+ if (e->ib) {+ imb_play_drawn(e->play, bk_time());+ }+ return e->ib;+}++BKImPlay *bk_image_entry_play(BKImEntry *e)+{+ return e->play;+}++void bk_image_entry_changed(BKImEntry *e)+{+ if (!e->req) {+ request(e, BK_IMAGE_PRIORITY_VISIBLE, true); /* a seek decodes now */+ }+ damage(e);+ arm(e);+ heard(e);+}++/** \} */+/* -------------------------------------------------------------------- *//** \name Settings, statistics, exit (BUI_kit.h)* \{ */diff --git a/kit/intern/image/draw.c b/kit/intern/image/draw.cindex 724ad53..bad4338 100644--- a/kit/intern/image/draw.c+++ b/kit/intern/image/draw.c@@ -1,7 +1,8 @@/* Images on nodes and canvas items: what a BKImageRef shows, fitted into a* box (CSS object-fit, the same bk_image_fit the decoder used), drawn with* BUI_raster_image. Sources go through the cache: drawn when decoded,- * requested at the box size otherwise (the box redraws on arrival).+ * requested at the box size otherwise (the box redraws on arrival). Videos+ * (video.c) show their current frame the same way.** SPDX-License-Identifier: GPL-2.0-or-later */@@ -71,6 +72,18 @@ void bk_image_ref_set_imbuf(BKImageRef *ref, const BKImBuf *ib)}}+void bk_image_ref_set_video(BKImageRef *ref, BKVideo *v)+{+ if (ref->kind == BK_IMREF_VIDEO && ref->u.video == v) {+ return;+ }+ bk_image_ref_clear(ref);+ if (v) {+ ref->kind = BK_IMREF_VIDEO;+ ref->u.video = v;+ }+}+void bk_image_ref_set_source(BKImageRef *ref, const char *src){if (ref->kind == BK_IMREF_SOURCE && src &&@@ -102,6 +115,9 @@ bool bk_image_ref_size(const BKImageRef *ref, int *r_w, int *r_h)*r_h = ib->y;return true;}+ if (ref->kind == BK_IMREF_VIDEO) {+ return bk_video_size(ref->u.video, r_w, r_h);+ }if (ref->kind == BK_IMREF_SOURCE && bk_image_source_info(ref->u.source, &info)) {*r_w = info.width;*r_h = info.height;@@ -116,10 +132,11 @@ bool bk_image_ref_resolve(const BKImageRef *ref, int fit, int frame, const float{const BKImageFit target = {fit, BK_MAX(1, (int)lroundf(box[2])), BK_MAX(1, (int)lroundf(box[3])),0.5f, 0.5f, BK_MAX(0, frame)};- if (ref->kind == BK_IMREF_SOURCE) {+ if (ref->kind == BK_IMREF_SOURCE || ref->kind == BK_IMREF_VIDEO) {const int rect[4] = {(int)floorf(box[0]), (int)floorf(box[1]), (int)ceilf(box[0] + box[2]),(int)ceilf(box[1] + box[3])};- r->ib = bk_image_cache_get(ref->u.source, &target, rect);+ r->ib = ref->kind == BK_IMREF_VIDEO ? bk_video_frame(ref->u.video, &target, rect) :+ bk_image_cache_get(ref->u.source, &target, rect);if (!r->ib) {return false;}@@ -142,7 +159,7 @@ bool bk_image_ref_resolve(const BKImageRef *ref, int fit, int frame, const floatvoid bk_image_draw_resolved(BuiRaster *r, const BKImageDraw *d, float alpha){- const BuiPixmap pm = {d->ib->rect, d->ib->x, d->ib->y, d->ib->key};+ const BuiPixmap pm = imb_pixmap(d->ib);BUI_raster_image(r, d->dst[0], d->dst[1], d->dst[2], d->dst[3], &pm, d->src, alpha, 0);}@@ -156,7 +173,7 @@ void bk_image_ref_draw(const BKImageRef *ref, int fit, int frame, const float bovoid bk_imbuf_draw(const BKImBuf *ib, float x, float y, float w, float h, float alpha){- const BuiPixmap pm = {ib->rect, ib->x, ib->y, ib->key};+ const BuiPixmap pm = imb_pixmap(ib);BUI_raster_image(bk_draw.r, x, y, w, h, &pm, NULL, alpha, 0);}@@ -246,20 +263,29 @@ const BKClass bk_cls_image = {.name = "image", .measure = image_measure, .draw =* holds straight RGBA (Dear ImGui's convention). */static int texture_put(const BKImBuf *ib, int frame, int tex){- const uint32_t *px = BK_imbuf_pixels(ib, frame);int tw, th;- if (!px || (tex && (!BUI_texture_size(tex, &tw, &th) || tw != ib->x || th != ib->y))) {+ if (!ib || frame < 0 || frame >= ib->frames ||+ (tex && (!BUI_texture_size(tex, &tw, &th) || tw != ib->x || th != ib->y)))+ {return 0;}- unsigned char *rgba = (unsigned char *)MEM_mallocN((size_t)ib->x * ib->y * 4, "texture rgba");- if (!rgba) {+ /* bands of rows through the stack: no allocation per frame (video textures) */+ uint8_t band[64 * 1024];+ const int step = (int)sizeof(band) / (ib->x * 4);+ if (step < 1) {return 0;}- imb_unpremultiply(rgba, px, ib->x * ib->y);- const int ok = tex ? BUI_texture_update(tex, 0, 0, ib->x, ib->y, rgba, 0) :- BUI_texture_create(ib->x, ib->y, rgba, 0);- MEM_freeN(rgba);- return ok;+ const int t = tex ? tex : BUI_texture_create(ib->x, ib->y, NULL, 0);+ for (int y = 0; t && y < ib->y; y += step) {+ const int n = BK_MIN(step, ib->y - y);+ if (!imb_rgba_rows(ib, frame, y, n, band) || !BUI_texture_update(t, 0, y, ib->x, n, band, 0)) {+ if (!tex) {+ BUI_texture_free(t);+ }+ return 0;+ }+ }+ return t;}int BK_imbuf_texture(const BKImBuf *ib, int frame)diff --git a/kit/intern/image/format_webp.c b/kit/intern/image/format_webp.cindex be1ceca..41cfcc6 100644--- a/kit/intern/image/format_webp.c+++ b/kit/intern/image/format_webp.c@@ -95,8 +95,12 @@ static void webp_close(void *state)MEM_freeN(a);}-static void *webp_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *r_loops)+static void *webp_open(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,+ BKImAnimInfo *r_info){+ (void)path;+ (void)want_w;+ (void)want_h;WebPAnimDecoderOptions opt;if (!WebPAnimDecoderOptionsInit(&opt)) {return NULL;@@ -114,14 +118,16 @@ static void *webp_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, intwebp_close(a);return NULL;}- *r_w = a->w = (int)info.canvas_width;- *r_h = a->h = (int)info.canvas_height;- *r_loops = (int)info.loop_count;+ r_info->w = r_info->src_w = a->w = (int)info.canvas_width;+ r_info->h = r_info->src_h = a->h = (int)info.canvas_height;+ r_info->format = BUI_PIXMAP_ARGB;+ r_info->loops = (int)info.loop_count;return a;}-static int webp_next(void *state, uint32_t *dst, int *r_delay)+static int webp_next(void *state, BKImBuf *dst, double *r_pts, double *r_dur){+ (void)r_pts;WebpAnim *a = (WebpAnim *)state;uint8_t *buf;int ts;@@ -131,17 +137,19 @@ static int webp_next(void *state, uint32_t *dst, int *r_delay)if (!WebPAnimDecoderGetNext(a->dec, &buf, &ts)) {return -1;}- memcpy(dst, buf, sizeof(uint32_t) * (size_t)a->w * a->h);- *r_delay = ts - a->prev_ts; /* timestamps are frame ends */+ memcpy(dst->rect, buf, sizeof(uint32_t) * (size_t)a->w * a->h);+ *r_dur = (ts - a->prev_ts) / 1000.0; /* timestamps (ms) are frame ends */a->prev_ts = ts;return 1;}-static void webp_rewind(void *state)+static bool webp_seek(void *state, double t){+ (void)t; /* from the top: anim.c skips to t */WebpAnim *a = (WebpAnim *)state;WebPAnimDecoderReset(a->dec);a->prev_ts = 0;+ return true;}static size_t webp_bytes(const void *state)@@ -150,6 +158,6 @@ static size_t webp_bytes(const void *state)return sizeof(*a) + 2 * sizeof(uint32_t) * (size_t)a->w * a->h; /* its canvas and previous */}-const BKImAnimCodec imb_webp_codec = {webp_open, webp_next, webp_rewind, webp_bytes, webp_close};+const BKImAnimCodec imb_webp_codec = {"webp", false, imb_webp_animated, webp_open, webp_next, webp_seek, webp_bytes, webp_close};/** \} */diff --git a/kit/intern/image/format_wuffs.c b/kit/intern/image/format_wuffs.cindex 19b9007..7acb33a 100644--- a/kit/intern/image/format_wuffs.c+++ b/kit/intern/image/format_wuffs.c@@ -272,8 +272,12 @@ static void gif_close(void *state)MEM_freeN(g);}-static void *gif_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *r_loops)+static void *gif_open(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,+ BKImAnimInfo *r_info){+ (void)path;+ (void)want_w;+ (void)want_h;GifAnim *g = (GifAnim *)MEM_callocN(sizeof(GifAnim), "GifAnim");g->mem = mem;g->size = size;@@ -292,8 +296,9 @@ static void *gif_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *gif_close(g);return NULL;}- *r_w = g->w;- *r_h = g->h;+ r_info->w = r_info->src_w = g->w;+ r_info->h = r_info->src_h = g->h;+ r_info->format = BUI_PIXMAP_ARGB;/* plays (0: forever): known once the frames were seen (no NETSCAPE block:* one play), so ask a decoder that skipped through them */wuffs_base__io_buffer src;@@ -301,19 +306,21 @@ static void *gif_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *wuffs_base__image_decoder *count = open_image(mem, size, &src, &ic);if (count) {count_frames(count, &src);- *r_loops = (int)wuffs_base__image_decoder__num_animation_loops(count);+ r_info->loops = (int)wuffs_base__image_decoder__num_animation_loops(count);MEM_freeN(count);}return g;}-static void gif_rewind(void *state)+static bool gif_seek(void *state, double t){- gif_start((GifAnim *)state);+ (void)t; /* from the top: anim.c skips to t */+ return gif_start((GifAnim *)state);}-static int gif_next(void *state, uint32_t *dst, int *r_delay)+static int gif_next(void *state, BKImBuf *dst, double *r_pts, double *r_dur){+ (void)r_pts;GifAnim *g = (GifAnim *)state;if (!g->dec) {return -1;@@ -343,10 +350,10 @@ static int gif_next(void *state, uint32_t *dst, int *r_delay)if (frame.repr && g->index == 0) {return -1; /* a bad first frame: no image (later bad frames end the animation) */}- *r_delay = (int)(wuffs_base__frame_config__duration(&fc) / WUFFS_BASE__FLICKS_PER_MILLISECOND);+ *r_dur = (double)wuffs_base__frame_config__duration(&fc) / (double)WUFFS_BASE__FLICKS_PER_SECOND;g->prev = fc;g->index++;- memcpy(dst, g->canvas, sizeof(uint32_t) * (size_t)g->w * g->h);+ memcpy(dst->rect, g->canvas, sizeof(uint32_t) * (size_t)g->w * g->h);return 1;}@@ -356,6 +363,6 @@ static size_t gif_bytes(const void *state)return sizeof(*g) + (size_t)g->wl + 2 * sizeof(uint32_t) * (size_t)g->w * g->h;}-const BKImAnimCodec imb_gif_codec = {gif_open, gif_next, gif_rewind, gif_bytes, gif_close};+const BKImAnimCodec imb_gif_codec = {"gif", false, imb_is_a_gif, gif_open, gif_next, gif_seek, gif_bytes, gif_close};/** \} */diff --git a/kit/intern/image/image.h b/kit/intern/image/image.hindex 1d56693..992cf4a 100644--- a/kit/intern/image/image.h+++ b/kit/intern/image/image.h@@ -40,7 +40,9 @@struct BKImBuf {int x, y; /* size in pixels (ImBuf::x, ::y) */int frames; /* >= 1 */- int ftype; /* BK_IMAGE_* it was decoded from */+ int ftype; /* BK_IMAGE_* it was decoded from (BK_IMAGE_VIDEO: a movie frame) */+ int format; /* BUI_PIXMAP_*: ARGB, or NV12 (one frame; `rect` holds its planes) */+ uint64_t stream; /* frames of one playing movie share it (one GPU texture); 0 else */int src_x, src_y; /* intrinsic size of the source (after EXIF orientation) */float crop[4]; /* the part to draw (x, y, w, h in these pixels): the visible* region of a fitted decode, sub-pixel exact; else all */@@ -56,6 +58,16 @@ struct BKImBuf {/* One block: header, delays, pixels (uninitialized). NULL when the size is* out of range or allocation fails. */BKImBuf *imb_alloc(int x, int y, int frames);+/* One NV12 frame (`format` BUI_PIXMAP_NV12_*; x and y even): luma, then chroma. */+BKImBuf *imb_alloc_nv12(int x, int y, int format);+/* A fresh unique key (BKImBuf keys, movie streams). */+uint64_t imb_new_key(void);+/* What the GPU renderers draw for `ib`. */+static inline BuiPixmap imb_pixmap(const BKImBuf *ib)+{+ const BuiPixmap pm = {ib->rect, ib->x, ib->y, ib->key, ib->format, ib->stream};+ return pm;+}static inline uint32_t *imb_frame(const BKImBuf *ib, int frame){return ib->rect + (size_t)frame * ib->x * ib->y;@@ -72,6 +84,8 @@ void imb_rekey(BKImBuf *ib);/* Straight RGBA (8 bits, any stride) <-> premultiplied 0xAARRGGBB rows. */void imb_premultiply_rgba(uint32_t *dst, const uint8_t *rgba, int n);void imb_unpremultiply(uint8_t *rgba, const uint32_t *src, int n);+/* Rows y0 .. y0 + rows of a frame as straight RGBA (any format). */+bool imb_rgba_rows(const BKImBuf *ib, int frame, int y0, int rows, uint8_t *dst);/* Where an image of intrinsic size iw x ih shows in a bw x bh box (CSS* object-fit / object-position): `dst` is the visible rect inside the box@@ -122,31 +136,71 @@ BKImBuf *imb_load(const uint8_t *mem, size_t size, const BKImageFit *target);/* Reads a whole file (MEM_mallocN'd), or its first `max` bytes (0: all). */uint8_t *imb_read_file(const char *path, size_t max, size_t *r_size);-/* -- anim.c: animations, one frame at a time ---------------------------------------- */+/* -- anim.c: animated media, one frame at a time --------------------------------------+ * GIF, animated WebP and video share one codec interface (Blender's anim /+ * MovieReader: open, next frame, seek, close) and one player (play.c). */++typedef struct BKImAnimInfo {+ int w, h; /* the frames' size: the canvas (in-memory codecs) or what a+ * streamed codec decodes to (about the size asked, even) */+ int src_w, src_h; /* intrinsic (display) size */+ int format; /* BUI_PIXMAP_* of the frames */+ int loops; /* plays the file asks for, 0 = forever */+ double duration; /* seconds, 0 when unknown */+} BKImAnimInfo;-/* What a format provides for animations: frames composited (disposal,- * blending) into a full-size premultiplied canvas copy. */typedef struct BKImAnimCodec {- void *(*open)(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *r_loops);- /* 1 with the next frame in `dst` (w * h), 0 past the last, -1 on errors */- int (*next)(void *state, uint32_t *dst, int *r_delay_ms);- void (*rewind)(void *state);- size_t (*bytes)(const void *state);+ const char *name;+ /* Streamed codecs (video) read the file themselves, by path, as it plays:+ * nothing is loaded whole. Others get the whole file in memory. */+ bool streamed;+ /* The file starts like one this codec reads (a few dozen bytes). */+ bool (*is_a)(const uint8_t *head, size_t size);+ /* `path` (streamed) or `mem` (kept by the caller while open). want_w x+ * want_h is the size frames are drawn at: a hint (video scales to it on+ * the decoder, in hardware where there is one). */+ void *(*open)(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,+ BKImAnimInfo *r_info);+ /* 1 with the next frame in `dst` (the info's size and format), its time+ * and duration in seconds (pts < 0: unknown, frames follow each other);+ * 0 past the last; -1 on errors. */+ int (*next)(void *state, BKImBuf *dst, double *r_pts, double *r_dur);+ /* The next frame is the one showing at `t`, or one before it (a keyframe;+ * in-memory codecs restart): anim.c skips forward to `t`. */+ bool (*seek)(void *state, double t);+ size_t (*bytes)(const void *state); /* decoder state (estimate for OS decoders) */void (*close)(void *state);} BKImAnimCodec;extern const BKImAnimCodec imb_gif_codec, imb_webp_codec;-/* The codec for an animation (GIF always, animated WebP). */+/* Video backends (movie_*.c), each compiled when the build has it. */+extern const BKImAnimCodec imb_movie_mf, imb_movie_avf, imb_movie_gst, imb_movie_dav1d,+ imb_movie_ffmpeg;+/* Those compiled in (movie.c: the build's choice), tried in order; NULL-ended. */+extern const BKImAnimCodec *const imb_movie_codecs[];+/* The first bytes look like a movie container (MP4/MOV, WebM/MKV, AVI, ASF,+ * Ogg, MPEG-PS/TS, FLV). */+bool imb_is_a_movie(const uint8_t *head, size_t size);+/* The in-memory codec for an animation (GIF always, animated WebP). */const BKImAnimCodec *imb_anim_codec(const uint8_t *mem, size_t size);typedef struct BKImAnim BKImAnim;/* Takes `file` (MEM-allocated; freed with the animation, or now on failure). */BKImAnim *imb_anim_open(uint8_t *file, size_t size, const BKImageFit *target);+/* A movie (streamed codecs; NULL when none reads the file). */+BKImAnim *imb_movie_open(const char *path, const BKImageFit *target);void imb_anim_free(BKImAnim *a);-int imb_anim_loops(const BKImAnim *a);+const BKImAnimInfo *imb_anim_info(const BKImAnim *a);int imb_anim_frames(const BKImAnim *a); /* from the header (at least what was decoded) */size_t imb_anim_bytes(const BKImAnim *a); /* its decoder state and file */-/* The next frame at the target size, wrapping (`r_wrapped`) after the last. */-BKImBuf *imb_anim_next(BKImAnim *a, int *r_delay_ms, bool *r_wrapped);+/* Frames to `target` from now on (a movie reopens its decoder at the new+ * size, where it was). False when it can't (the old size stays). */+bool imb_anim_retarget(BKImAnim *a, const BKImageFit *target);+/* The next frame at the target size, wrapping (`r_wrapped`) after the last.+ * `reuse` (taken; NULL for none): a frame no longer shown, which a movie+ * decodes into when it fits (no allocation per frame). */+BKImBuf *imb_anim_next(BKImAnim *a, BKImBuf *reuse, double *r_pts, double *r_dur, bool *r_wrapped);+/* The next imb_anim_next returns the frame showing at `t` seconds. */+void imb_anim_seek(BKImAnim *a, double t);/* Frame `frame` alone, or every frame (synchronous API; at most 128 MiB). */BKImBuf *imb_anim_frame(const uint8_t *mem, size_t size, const BKImageFit *target, int frame);BKImBuf *imb_anim_all(const uint8_t *mem, size_t size, const BKImageFit *target);@@ -218,20 +272,60 @@ double bk_image_next_due(BKContext *ctx);const BKImBuf *bk_image_cache_get(BKImageSource *s, const BKImageFit *target, const int rect[4]);void bk_image_cache_prefetch(BKImageSource *s, const BKImageFit *target);-/* play.c: the frame clock of a playing animation (owned by its cache entry). */-typedef struct BKImPlay BKImPlay;-BKImPlay *imb_play_new(BKImAnim *anim, int delay); /* takes `anim`; shows its first frame */+/* A private entry: one player's (a <video>), outside the shared table and+ * the LRU (never evicted), freed by its owner (video.c). It loads at its+ * first draw, with `play` (taken) as its clock; `notify` hears every state+ * change of the clock (loaded, ended, error) on the UI thread. */+typedef struct BKImEntry BKImEntry;+typedef struct BKImPlay BKImPlay; /* play.c */+BKImEntry *bk_image_entry_new(BKImageSource *s, BKImPlay *play, void (*notify)(void *owner),+ void *owner);+void bk_image_entry_free(BKImEntry *e);+/* As bk_image_cache_get; the entry follows the size it is drawn at. */+const BKImBuf *bk_image_entry_get(BKImEntry *e, const BKImageFit *target, const int rect[4]);+BKImPlay *bk_image_entry_play(BKImEntry *e);+/* A control changed the clock: step now, redraw where it shows. */+void bk_image_entry_changed(BKImEntry *e);++/* play.c: the one clock of everything that plays (GIF, WebP, video), owned+ * by its cache entry. Frames advance only after the shown one was drawn. */+typedef enum BKImPlayState {+ BK_PLAY_LOADING = 0, /* no frame yet */+ BK_PLAY_PLAYING,+ BK_PLAY_PAUSED,+ BK_PLAY_ENDED, /* the last play's last frame shows */+ BK_PLAY_ERROR,+} BKImPlayState;+/* `loops` overrides the file's play count (-1: the file's; 0: forever). */+BKImPlay *imb_play_new(bool paused, int loops);void imb_play_free(BKImPlay *p);size_t imb_play_bytes(const BKImPlay *p);-/* The decoder for the next step request (NULL: nothing to decode now). */-BKImAnim *imb_play_take_step(BKImPlay *p);+/* The first frame arrived with its decoder (taken). */+void imb_play_start(BKImPlay *p, BKImAnim *anim, double pts, double dur);+void imb_play_fail(BKImPlay *p); /* the file didn't open */+/* A step request to make: the decoder (NULL: nothing to decode now), the+ * buffer to decode into (or NULL) and where to seek first (< 0: no seek).+ * The step owns them until it comes back (step_done / step_cancelled). */+BKImAnim *imb_play_take_step(BKImPlay *p, BKImBuf **r_reuse, double *r_seek);/* The step came back: the decoder, and the next frame (NULL: an error). */-void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, int delay, bool wrapped);-void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim); /* the decoder back, no frame */+void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, double pts, double dur,+ bool wrapped);+void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim, BKImBuf *reuse); /* nothing decoded */void imb_play_drawn(BKImPlay *p, double now); /* the shown frame reached the screen */double imb_play_due(const BKImPlay *p); /* when to advance (0: not waiting on time) */-/* The frame to show from `now` (the caller frees the old one), or NULL. */+/* The frame to show from `now`, or NULL; the one it replaces goes back+ * through imb_play_recycle. */BKImBuf *imb_play_advance(BKImPlay *p, double now);+void imb_play_recycle(BKImPlay *p, BKImBuf *old); /* kept for a movie's next decode, else freed */+/* Controls (the <video> element's). */+void imb_play_pause(BKImPlay *p, bool paused);+void imb_play_seek(BKImPlay *p, double t);+void imb_play_set_rate(BKImPlay *p, double rate); /* > 0 */+void imb_play_set_loops(BKImPlay *p, int loops); /* 0: forever, -1: the file's */+BKImPlayState imb_play_state(const BKImPlay *p);+double imb_play_time(const BKImPlay *p); /* seconds, of the frame shown */+double imb_play_duration(const BKImPlay *p); /* seconds, 0 when unknown */+bool imb_play_size(const BKImPlay *p, int *r_w, int *r_h); /* intrinsic, once loaded *//* draw.c: what a node or canvas item shows. One of an owned copy (set from* RGBA), a borrowed BKImBuf (the host keeps it alive) or a source decoded@@ -241,6 +335,7 @@ typedef enum BKImageRefKind {BK_IMREF_OWNED,BK_IMREF_BORROWED,BK_IMREF_SOURCE,+ BK_IMREF_VIDEO, /* a BKVideo (borrowed: its owner frees it) */} BKImageRefKind;typedef struct BKImageRef {BKImageRefKind kind;@@ -248,6 +343,7 @@ typedef struct BKImageRef {BKImBuf *owned;const BKImBuf *borrowed;BKImageSource *source;+ BKVideo *video;} u;} BKImageRef;void bk_image_ref_clear(BKImageRef *ref);@@ -255,6 +351,12 @@ void bk_image_ref_clear(BKImageRef *ref);void bk_image_ref_set_rgba(BKImageRef *ref, const unsigned char *rgba, int w, int h);void bk_image_ref_set_imbuf(BKImageRef *ref, const BKImBuf *ib);void bk_image_ref_set_source(BKImageRef *ref, const char *src);+void bk_image_ref_set_video(BKImageRef *ref, BKVideo *v);++/* video.c: a BKVideo's frame for drawing at `target` (NULL until the first+ * frame; then its poster, if any, is what shows), and its intrinsic size. */+const BKImBuf *bk_video_frame(BKVideo *v, const BKImageFit *target, const int rect[4]);+bool bk_video_size(const BKVideo *v, int *r_w, int *r_h);/* Intrinsic size (a source reads its header once). */bool bk_image_ref_size(const BKImageRef *ref, int *r_w, int *r_h);@@ -284,9 +386,13 @@ struct BKImRequest {BKImBuf *result; /* NULL when decoding failed */void *entry; /* the cache entry waiting for it (cache.c) */BKImAnim *anim; /* a step: the animation to decode the next frame of; a- * first load: set when the file plays (frames > 1) */- int delay; /* the result's delay (ms) and whether it starts a new play */- bool wrapped;+ * first load: set when the file plays (frames > 1, movies) */+ BKImBuf *reuse; /* a step: the frame to decode into (taken) */+ double seek; /* a step: seek there first (seconds; < 0: no seek) */+ bool step; /* the next frame of something playing (else a first load) */+ bool movie; /* a first load of a movie (streamed codecs, imb_movie_open) */+ double pts, dur; /* the result's time and duration (seconds), and whether */+ bool wrapped; /* it starts a new play */};enum { BK_IMREQ_QUEUED, BK_IMREQ_RUNNING, BK_IMREQ_DONE, BK_IMREQ_CANCELLED };/* Queue `req` (ownership moves to the loader until it comes back). */diff --git a/kit/intern/image/imbuf.c b/kit/intern/image/imbuf.cindex 786c05f..0e48a8b 100644--- a/kit/intern/image/imbuf.c+++ b/kit/intern/image/imbuf.c@@ -19,7 +19,7 @@/** \name Allocation* \{ */-static uint64_t next_key(void)+uint64_t imb_new_key(void){static volatile uint64_t key = 0;#ifdef _MSC_VER@@ -48,7 +48,7 @@ BKImBuf *imb_alloc(int x, int y, int frames)ib->x = ib->src_x = x;ib->y = ib->src_y = y;ib->frames = frames;- ib->key = next_key();+ ib->key = imb_new_key();ib->size = size;ib->delays = frames > 1 ? (int *)(ib + 1) : NULL;ib->rect = (uint32_t *)((char *)ib + head);@@ -57,9 +57,33 @@ BKImBuf *imb_alloc(int x, int y, int frames)return ib;}+BKImBuf *imb_alloc_nv12(int x, int y, int format)+{+ if (x <= 0 || y <= 0 || (x | y) & 1 || x > BK_IMB_MAX_SIDE || y > BK_IMB_MAX_SIDE) {+ return NULL;+ }+ const size_t head = (sizeof(BKImBuf) + 15) & ~(size_t)15;+ const size_t size = head + ((size_t)x * y * 3 / 2 + 3) / 4 * 4;+ BKImBuf *ib = (BKImBuf *)MEM_mallocN(size, "BKImBuf NV12");+ if (!ib) {+ return NULL;+ }+ memset(ib, 0, sizeof(*ib));+ ib->x = ib->src_x = x;+ ib->y = ib->src_y = y;+ ib->frames = 1;+ ib->format = format;+ ib->key = imb_new_key();+ ib->size = size;+ ib->rect = (uint32_t *)((char *)ib + head);+ ib->crop[2] = (float)x;+ ib->crop[3] = (float)y;+ return ib;+}+void imb_rekey(BKImBuf *ib){- ib->key = next_key();+ ib->key = imb_new_key();}/* A new buffer of w x h carrying `ib`'s metadata (frames, delays, source). */@@ -80,8 +104,8 @@ static BKImBuf *imb_alloc_like(const BKImBuf *ib, int w, int h)void BK_imbuf_free(BKImBuf *ib){- if (ib && (ib->x > BUI_IMAGE_ATLAS_MAX || ib->y > BUI_IMAGE_ATLAS_MAX)) {- BUI_image_forget(ib->key); /* its own GPU texture goes with it */+ if (ib && !ib->stream && (ib->x > BUI_IMAGE_ATLAS_MAX || ib->y > BUI_IMAGE_ATLAS_MAX)) {+ BUI_image_forget(ib->key); /* its own GPU texture goes with it (a stream's, with its movie) */}MEM_freeN(ib); /* one block (MEM_freeN(NULL) is a no-op) */}@@ -401,7 +425,9 @@ int BK_imbuf_format(const BKImBuf *ib)const uint32_t *BK_imbuf_pixels(const BKImBuf *ib, int frame){- return ib && frame >= 0 && frame < ib->frames ? imb_frame(ib, frame) : NULL;+ return ib && ib->format == BUI_PIXMAP_ARGB && frame >= 0 && frame < ib->frames ?+ imb_frame(ib, frame) :+ NULL;}int BK_imbuf_delay(const BKImBuf *ib, int frame)@@ -414,14 +440,31 @@ int BK_imbuf_loops(const BKImBuf *ib)return ib ? ib->loops : 0;}-int BK_imbuf_rgba(const BKImBuf *ib, int frame, unsigned char *dst)+bool imb_rgba_rows(const BKImBuf *ib, int frame, int y0, int rows, uint8_t *dst){- const uint32_t *px = BK_imbuf_pixels(ib, frame);- if (!px || !dst) {- return 0;+ if (ib->format == BUI_PIXMAP_ARGB) {+ imb_unpremultiply(dst, imb_frame(ib, frame) + (size_t)y0 * ib->x, ib->x * rows);+ return true;+ }+ /* a video frame: converted a band of rows at a time (no allocation) */+ const BuiPixmap pm = imb_pixmap(ib);+ uint32_t band[8192];+ const int cap = (int)(sizeof(band) / sizeof(band[0]));+ if (ib->x > cap) {+ return false; /* wider than a band: not a size movies decode to */+ }+ const int step = cap / ib->x;+ for (int y = y0; y < y0 + rows; y += step) {+ const int n = BK_MIN(step, y0 + rows - y);+ BUI_pixmap_read(&pm, y, n, band);+ imb_unpremultiply(dst + (size_t)(y - y0) * ib->x * 4, band, ib->x * n);}- imb_unpremultiply(dst, px, ib->x * ib->y);- return 1;+ return true;+}++int BK_imbuf_rgba(const BKImBuf *ib, int frame, unsigned char *dst)+{+ return ib && dst && frame >= 0 && frame < ib->frames && imb_rgba_rows(ib, frame, 0, ib->y, dst);}BKImBuf *BK_imbuf_from_rgba(const unsigned char *rgba, int w, int h, int stride)diff --git a/kit/intern/image/loader.c b/kit/intern/image/loader.cindex 4dd6466..38c09bc 100644--- a/kit/intern/image/loader.c+++ b/kit/intern/image/loader.c@@ -81,8 +81,25 @@ void bk_loader_notify(void)/* What a request decodes (on a worker). */static BKImBuf *decode(BKImRequest *req){- if (req->anim) { /* the next frame of a playing animation */- return imb_anim_next(req->anim, &req->delay, &req->wrapped);+ if (req->anim) { /* the next frame of what plays: at the size now drawn, sought first */+ imb_anim_retarget(req->anim, &req->target);+ if (req->seek >= 0.0) {+ imb_anim_seek(req->anim, req->seek);+ }+ BKImBuf *reuse = req->reuse;+ req->reuse = NULL;+ return imb_anim_next(req->anim, reuse, &req->pts, &req->dur, &req->wrapped);+ }+ if (req->movie) { /* a video: streamed from the file, the decoder kept */+ BKImAnim *a = imb_movie_open(bk_image_source_path(req->source), &req->target);+ BKImBuf *ib = a ? imb_anim_next(a, NULL, &req->pts, &req->dur, &req->wrapped) : NULL;+ if (ib) {+ req->anim = a;+ }+ else {+ imb_anim_free(a);+ }+ return ib;}size_t size;uint8_t *mem = imb_read_file(bk_image_source_path(req->source), 0, &size);@@ -94,7 +111,7 @@ static BKImBuf *decode(BKImRequest *req)/* an animation that plays: its first frame now, the decoder kept (it owns* the file) for the next ones; one frame is just a still */BKImAnim *a = imb_anim_open(mem, size, &req->target);- BKImBuf *ib = a ? imb_anim_next(a, &req->delay, &req->wrapped) : NULL;+ BKImBuf *ib = a ? imb_anim_next(a, NULL, &req->pts, &req->dur, &req->wrapped) : NULL;if (ib && imb_anim_frames(a) > 1) {req->anim = a;}diff --git a/kit/intern/image/movie.c b/kit/intern/image/movie.cnew file mode 100644index 0000000..4054edc--- /dev/null+++ b/kit/intern/image/movie.c@@ -0,0 +1,75 @@+/* Movies: the video backends this build has (chosen at build time from the+ * feature table, features.cmake: BUI_VIDEO_*), tried in order behind the one+ * codec interface (BKImAnimCodec), and what a movie file looks like.+ *+ * movie_mf.c Windows Media Foundation (the OS decoders, DXVA)+ * movie_avf.m macOS AVFoundation / VideoToolbox+ * movie_gst.c Linux GStreamer (VA-API through its plugins)+ * movie_dav1d.c AV1 with dav1d and a small MP4 / WebM demuxer (optional)+ * movie_ffmpeg.c FFmpeg (optional)+ *+ * Every backend loads its system libraries at run time (LoadLibrary /+ * dlopen / weak frameworks): a machine without them starts normally and+ * just has no video.+ *+ * SPDX-License-Identifier: GPL-2.0-or-later */++#include <string.h>++#include "image.h"++bool imb_is_a_movie(const uint8_t *m, size_t n)+{+ static const uint8_t ebml[4] = {0x1A, 0x45, 0xDF, 0xA3}; /* WebM, Matroska */+ static const uint8_t asf[4] = {0x30, 0x26, 0xB2, 0x75}; /* WMV, ASF */+ static const uint8_t mpeg_ps[4] = {0x00, 0x00, 0x01, 0xBA}; /* MPEG program stream */+ return (n >= 12 && memcmp(m + 4, "ftyp", 4) == 0 && memcmp(m + 8, "avif", 4) != 0 &&+ memcmp(m + 8, "heic", 4) != 0) || /* MP4, MOV, M4V, 3GP (not AVIF/HEIC stills) */+ (n >= 8 && (memcmp(m + 4, "moov", 4) == 0 || memcmp(m + 4, "mdat", 4) == 0 ||+ memcmp(m + 4, "wide", 4) == 0 || memcmp(m + 4, "free", 4) == 0)) ||+ (n >= 4 && (memcmp(m, ebml, 4) == 0 || memcmp(m, asf, 4) == 0 ||+ memcmp(m, mpeg_ps, 4) == 0 || memcmp(m, "OggS", 4) == 0)) ||+ (n >= 12 && memcmp(m, "RIFF", 4) == 0 && memcmp(m + 8, "AVI ", 4) == 0) ||+ (n >= 3 && memcmp(m, "FLV", 3) == 0) || (n >= 1 && m[0] == 0x47 && n >= 64); /* TS */+}++const BKImAnimCodec *const imb_movie_codecs[] = {+#ifdef BUI_VIDEO_DAV1D+ &imb_movie_dav1d, /* AV1 in software, before the OS (which may lack AV1) */+#endif+#ifdef BUI_VIDEO_MF+ &imb_movie_mf,+#endif+#ifdef BUI_VIDEO_AVF+ &imb_movie_avf,+#endif+#ifdef BUI_VIDEO_GST+ &imb_movie_gst,+#endif+#ifdef BUI_VIDEO_FFMPEG+ &imb_movie_ffmpeg,+#endif+ NULL,+};++const char *BK_video_backends(void)+{+ static char names[96];+ if (!names[0]) {+ size_t n = 0;+ for (int i = 0; imb_movie_codecs[i]; i++) {+ const char *s = imb_movie_codecs[i]->name;+ const size_t len = strlen(s);+ if (n + len + 2 >= sizeof(names)) {+ break;+ }+ if (n) {+ names[n++] = ',';+ }+ memcpy(names + n, s, len);+ n += len;+ }+ names[n] = '\0';+ }+ return names;+}diff --git a/kit/intern/image/movie_mf.c b/kit/intern/image/movie_mf.cnew file mode 100644index 0000000..3cf32a6--- /dev/null+++ b/kit/intern/image/movie_mf.c@@ -0,0 +1,420 @@+/* Video on Windows: Media Foundation's source reader (the OS demuxers and+ * decoders: H.264, HEVC, VP9 and AV1 where their extensions are installed,+ * MPEG-4, WMV), with DXVA hardware decoding through a shared Direct3D 11+ * device and the reader's video processor scaling to the size drawn on+ * the GPU (MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING). Frames come+ * back as NV12 at about the size drawn, copied into the frame anim.c gives.+ *+ * mfplat.dll, mfreadwrite.dll, d3d11.dll and ole32.dll are loaded on first+ * use: Windows "N" editions (no Media Feature Pack) start normally and just+ * have no video. The reader is free-threaded (COM MTA): one step at a time+ * runs on whichever worker takes it.+ *+ * SPDX-License-Identifier: GPL-2.0-or-later */++#ifdef BUI_VIDEO_MF++# define COBJMACROS+# define WIN32_LEAN_AND_MEAN+# include <windows.h>++# include <initguid.h> /* the GUIDs below, defined here: nothing links mfuuid.lib */++# include <d3d11.h>+# include <mfapi.h>+# include <mferror.h>+# include <mfidl.h>+# include <mfreadwrite.h>++# include <string.h>++# include "MEM_guardedalloc.h"++# include "image.h"++/* -------------------------------------------------------------------- */+/** \name The system libraries, loaded once+ * \{ */++typedef HRESULT(WINAPI *MFStartupFn)(ULONG, DWORD);+typedef HRESULT(WINAPI *MFCreateAttributesFn)(IMFAttributes **, UINT32);+typedef HRESULT(WINAPI *MFCreateMediaTypeFn)(IMFMediaType **);+typedef HRESULT(WINAPI *MFCreateDXGIDeviceManagerFn)(UINT *, IMFDXGIDeviceManager **);+typedef HRESULT(WINAPI *MFCreateSourceReaderFromURLFn)(LPCWSTR, IMFAttributes *,+ IMFSourceReader **);+typedef HRESULT(WINAPI *CoInitializeExFn)(LPVOID, DWORD);+typedef HRESULT(WINAPI *D3D11CreateDeviceFn)(IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT,+ const D3D_FEATURE_LEVEL *, UINT, UINT,+ ID3D11Device **, D3D_FEATURE_LEVEL *,+ ID3D11DeviceContext **);++static struct {+ INIT_ONCE once;+ bool ok;+ MFCreateAttributesFn create_attributes;+ MFCreateMediaTypeFn create_media_type;+ MFCreateSourceReaderFromURLFn create_reader;+ CoInitializeExFn co_init;+ /* DXVA: one device for every reader (NULL: software decoding) */+ IMFDXGIDeviceManager *manager;+} MF = {INIT_ONCE_STATIC_INIT};++static void *sym(HMODULE dll, const char *name)+{+ return dll ? (void *)GetProcAddress(dll, name) : NULL;+}++/* A video device for DXVA decoding and the reader's video processor. */+static void dxva_init(HMODULE mfplat)+{+ MFCreateDXGIDeviceManagerFn create_manager = (MFCreateDXGIDeviceManagerFn)sym(+ mfplat, "MFCreateDXGIDeviceManager");+ D3D11CreateDeviceFn create_device = (D3D11CreateDeviceFn)sym(LoadLibraryW(L"d3d11.dll"),+ "D3D11CreateDevice");+ ID3D11Device *dev = NULL;+ UINT token = 0;+ if (!create_manager || !create_device ||+ FAILED(create_device(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL,+ D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT,+ NULL, 0, D3D11_SDK_VERSION, &dev, NULL, NULL)))+ {+ return;+ }+ /* readers on several workers share it */+ ID3D10Multithread *mt = NULL;+ if (SUCCEEDED(ID3D11Device_QueryInterface(dev, &IID_ID3D10Multithread, (void **)&mt))) {+ ID3D10Multithread_SetMultithreadProtected(mt, TRUE);+ ID3D10Multithread_Release(mt);+ }+ if (SUCCEEDED(create_manager(&token, &MF.manager)) &&+ FAILED(IMFDXGIDeviceManager_ResetDevice(MF.manager, (IUnknown *)dev, token)))+ {+ IMFDXGIDeviceManager_Release(MF.manager);+ MF.manager = NULL;+ }+ ID3D11Device_Release(dev); /* the manager keeps it for the process */+}++static BOOL CALLBACK load(PINIT_ONCE once, PVOID param, PVOID *ctx)+{+ (void)once;+ (void)param;+ (void)ctx;+ HMODULE mfplat = LoadLibraryW(L"mfplat.dll"), mfrw = LoadLibraryW(L"mfreadwrite.dll");+ MFStartupFn startup = (MFStartupFn)sym(mfplat, "MFStartup");+ MF.create_attributes = (MFCreateAttributesFn)sym(mfplat, "MFCreateAttributes");+ MF.create_media_type = (MFCreateMediaTypeFn)sym(mfplat, "MFCreateMediaType");+ MF.create_reader = (MFCreateSourceReaderFromURLFn)sym(mfrw, "MFCreateSourceReaderFromURL");+ MF.co_init = (CoInitializeExFn)sym(LoadLibraryW(L"ole32.dll"), "CoInitializeEx");+ if (!startup || !MF.create_attributes || !MF.create_media_type || !MF.create_reader ||+ !MF.co_init)+ {+ return TRUE; /* no Media Foundation: no video, nothing else changes */+ }+ MF.co_init(NULL, COINIT_MULTITHREADED);+ MF.ok = SUCCEEDED(startup(MF_VERSION, MFSTARTUP_LITE)); /* no network sources */+ if (MF.ok) {+ dxva_init(mfplat);+ }+ return TRUE;+}++/* Media Foundation, ready on this thread (COM MTA). */+static bool mf_ready(void)+{+# ifdef _MSC_VER+ static __declspec(thread) bool com;+# else+ static _Thread_local bool com;+# endif+ InitOnceExecuteOnce(&MF.once, load, NULL, NULL);+ if (MF.ok && !com) {+ const HRESULT hr = MF.co_init(NULL, COINIT_MULTITHREADED);+ com = SUCCEEDED(hr) || hr == RPC_E_CHANGED_MODE; /* an STA host thread works too */+ }+ return MF.ok && com;+}++/** \} */++/* -------------------------------------------------------------------- */+/** \name The codec+ * \{ */++typedef struct MFMovie {+ IMFSourceReader *reader;+ int w, h; /* output frame (NV12) */+ double frame_dur; /* from the frame rate, for samples without a duration */+} MFMovie;++# define VIDEO ((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM)++static void mf_close(void *state)+{+ MFMovie *m = (MFMovie *)state;+ if (m->reader) {+ IMFSourceReader_Release(m->reader);+ }+ MEM_freeN(m);+}++/* The output's size, as the reader set it up. */+static bool output_size(MFMovie *m)+{+ IMFMediaType *t = NULL;+ UINT64 size = 0;+ if (FAILED(IMFSourceReader_GetCurrentMediaType(m->reader, VIDEO, &t))) {+ return false;+ }+ const bool ok = SUCCEEDED(IMFMediaType_GetUINT64(t, &MF_MT_FRAME_SIZE, &size));+ IMFMediaType_Release(t);+ m->w = (int)(size >> 32);+ m->h = (int)(size & 0xFFFFFFFF);+ return ok;+}++/* NV12 at out_w x out_h (the reader's video processor scales and converts). */+static bool set_output(MFMovie *m, int out_w, int out_h, bool interlaced)+{+ IMFMediaType *t = NULL;+ if (FAILED(MF.create_media_type(&t))) {+ return false;+ }+ IMFMediaType_SetGUID(t, &MF_MT_MAJOR_TYPE, &MFMediaType_Video);+ IMFMediaType_SetGUID(t, &MF_MT_SUBTYPE, &MFVideoFormat_NV12);+ IMFMediaType_SetUINT64(t, &MF_MT_FRAME_SIZE, (UINT64)out_w << 32 | (UINT32)out_h);+ IMFMediaType_SetUINT64(t, &MF_MT_PIXEL_ASPECT_RATIO, (UINT64)1 << 32 | 1);+ if (interlaced) {+ IMFMediaType_SetUINT32(t, &MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); /* deinterlace */+ }+ const HRESULT hr = IMFSourceReader_SetCurrentMediaType(m->reader, VIDEO, NULL, t);+ IMFMediaType_Release(t);+ return SUCCEEDED(hr) && output_size(m);+}++static wchar_t *wide_path(const char *path)+{+ const int n = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);+ wchar_t *w = n > 0 ? (wchar_t *)MEM_mallocN(sizeof(wchar_t) * (size_t)n, "movie path") : NULL;+ if (w && !MultiByteToWideChar(CP_UTF8, 0, path, -1, w, n)) {+ MEM_freeN(w);+ w = NULL;+ }+ return w;+}++static IMFSourceReader *open_reader(const char *path)+{+ IMFAttributes *attr = NULL;+ IMFSourceReader *reader = NULL;+ wchar_t *wpath = wide_path(path);+ if (wpath && SUCCEEDED(MF.create_attributes(&attr, 4))) {+ IMFAttributes_SetUINT32(attr, &MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, TRUE);+ IMFAttributes_SetUINT32(attr, &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE);+ if (MF.manager) {+ IMFAttributes_SetUnknown(attr, &MF_SOURCE_READER_D3D_MANAGER, (IUnknown *)MF.manager);+ }+ if (FAILED(MF.create_reader(wpath, attr, &reader))) {+ reader = NULL;+ }+ IMFAttributes_Release(attr);+ }+ MEM_freeN(wpath);+ return reader;+}++static void *mf_open(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,+ BKImAnimInfo *r_info)+{+ (void)mem;+ (void)size;+ if (!path || !mf_ready()) {+ return NULL;+ }+ MFMovie *m = (MFMovie *)MEM_callocN(sizeof(MFMovie), "MFMovie");+ m->reader = open_reader(path);+ IMFMediaType *native = NULL;+ if (!m->reader ||+ FAILED(IMFSourceReader_SetStreamSelection(m->reader, (DWORD)MF_SOURCE_READER_ALL_STREAMS,+ FALSE)) ||+ FAILED(IMFSourceReader_SetStreamSelection(m->reader, VIDEO, TRUE)) ||+ FAILED(IMFSourceReader_GetNativeMediaType(m->reader, VIDEO, 0, &native)))+ {+ mf_close(m);+ return NULL;+ }+ /* the intrinsic size: the coded frame through its pixel aspect ratio */+ UINT64 fs = 0, par = 0, rate = 0;+ UINT32 matrix = 0, interlace = MFVideoInterlace_Progressive;+ IMFMediaType_GetUINT64(native, &MF_MT_FRAME_SIZE, &fs);+ const bool has_par = SUCCEEDED(IMFMediaType_GetUINT64(native, &MF_MT_PIXEL_ASPECT_RATIO, &par));+ IMFMediaType_GetUINT64(native, &MF_MT_FRAME_RATE, &rate);+ IMFMediaType_GetUINT32(native, &MF_MT_YUV_MATRIX, &matrix);+ IMFMediaType_GetUINT32(native, &MF_MT_INTERLACE_MODE, &interlace);+ IMFMediaType_Release(native);+ int src_w = (int)(fs >> 32), src_h = (int)(fs & 0xFFFFFFFF);+ const UINT32 pn = (UINT32)(par >> 32), pd = (UINT32)par;+ if (has_par && pn && pd && pn != pd) {+ src_w = (int)((double)src_w * pn / pd + 0.5);+ }+ if (src_w <= 0 || src_h <= 0 || src_w > BK_IMB_MAX_SIDE || src_h > BK_IMB_MAX_SIDE) {+ mf_close(m);+ return NULL;+ }+ /* decode at the size asked (never above the source), even for NV12 */+ int out_w = want_w > 0 ? BK_MIN(want_w, src_w) : src_w;+ int out_h = want_h > 0 ? BK_MIN(want_h, src_h) : src_h;+ out_w = BK_MAX(2, out_w & ~1);+ out_h = BK_MAX(2, out_h & ~1);+ const bool interlaced = interlace != MFVideoInterlace_Progressive;+ if (!set_output(m, out_w, out_h, interlaced) &&+ !set_output(m, BK_MAX(2, src_w & ~1), BK_MAX(2, src_h & ~1), interlaced))+ {+ mf_close(m); /* no NV12 output: no decoder for this format */+ return NULL;+ }+ const UINT32 rn = (UINT32)(rate >> 32), rd = (UINT32)rate;+ m->frame_dur = rn && rd ? (double)rd / rn : 1.0 / 30.0;+ PROPVARIANT dur;+ memset(&dur, 0, sizeof(dur)); /* VT_UI8: nothing to clear */+ r_info->duration =+ SUCCEEDED(IMFSourceReader_GetPresentationAttribute(+ m->reader, (DWORD)MF_SOURCE_READER_MEDIASOURCE, &MF_PD_DURATION, &dur)) &&+ dur.vt == VT_UI8 ?+ (double)dur.uhVal.QuadPart / 1e7 :+ 0.0;+ r_info->w = m->w;+ r_info->h = m->h;+ r_info->src_w = src_w;+ r_info->src_h = src_h;+ r_info->loops = 1;+ r_info->format = matrix == MFVideoTransferMatrix_BT601 ? BUI_PIXMAP_NV12_BT601 :+ matrix == MFVideoTransferMatrix_BT709 ? BUI_PIXMAP_NV12_BT709 :+ src_h >= 720 ? BUI_PIXMAP_NV12_BT709 :+ BUI_PIXMAP_NV12_BT601;+ return m;+}++/* The frame's planes into `dst` (NV12, its size). */+static bool copy_frame(MFMovie *m, IMFSample *sample, BKImBuf *dst)+{+ IMFMediaBuffer *buf = NULL;+ if (FAILED(IMFSample_GetBufferByIndex(sample, 0, &buf))) {+ return false;+ }+ /* rows of a 2D buffer are `pitch` apart; chroma starts after the plane's+ * allocated height (a GPU surface may be taller than the frame) */+ IMF2DBuffer *b2 = NULL;+ IMFDXGIBuffer *dx = NULL;+ BYTE *scan0 = NULL;+ LONG pitch = 0;+ int plane_h = m->h;+ bool locked2d = false;+ if (SUCCEEDED(IMFMediaBuffer_QueryInterface(buf, &IID_IMFDXGIBuffer, (void **)&dx))) {+ ID3D11Texture2D *tex = NULL;+ if (SUCCEEDED(IMFDXGIBuffer_GetResource(dx, &IID_ID3D11Texture2D, (void **)&tex))) {+ D3D11_TEXTURE2D_DESC td;+ ID3D11Texture2D_GetDesc(tex, &td);+ plane_h = (int)td.Height;+ ID3D11Texture2D_Release(tex);+ }+ IMFDXGIBuffer_Release(dx);+ }+ if (SUCCEEDED(IMFMediaBuffer_QueryInterface(buf, &IID_IMF2DBuffer, (void **)&b2))) {+ locked2d = SUCCEEDED(IMF2DBuffer_Lock2D(b2, &scan0, &pitch));+ }+ DWORD len = 0;+ if (!locked2d && SUCCEEDED(IMFMediaBuffer_Lock(buf, &scan0, NULL, &len))) {+ pitch = m->w; /* a contiguous buffer: packed rows */+ }+ const bool ok = scan0 && pitch >= dst->x && plane_h >= dst->y;+ if (ok) {+ uint8_t *y = (uint8_t *)dst->rect, *uv = y + (size_t)dst->x * dst->y;+ const BYTE *sy = scan0, *suv = scan0 + (size_t)pitch * plane_h;+ for (int r = 0; r < dst->y; r++) {+ memcpy(y + (size_t)r * dst->x, sy + (size_t)r * pitch, (size_t)dst->x);+ }+ for (int r = 0; r < dst->y / 2; r++) {+ memcpy(uv + (size_t)r * dst->x, suv + (size_t)r * pitch, (size_t)dst->x);+ }+ }+ if (locked2d) {+ IMF2DBuffer_Unlock2D(b2);+ }+ else if (scan0) {+ IMFMediaBuffer_Unlock(buf);+ }+ if (b2) {+ IMF2DBuffer_Release(b2);+ }+ IMFMediaBuffer_Release(buf);+ return ok;+}++static int mf_next(void *state, BKImBuf *dst, double *r_pts, double *r_dur)+{+ MFMovie *m = (MFMovie *)state;+ for (int gaps = 0; gaps < 64; gaps++) {+ DWORD stream = 0, flags = 0;+ LONGLONG ts = 0;+ IMFSample *sample = NULL;+ if (FAILED(IMFSourceReader_ReadSample(m->reader, VIDEO, 0, &stream, &flags, &ts, &sample))) {+ return -1;+ }+ if (flags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED) {+ const int w = m->w, h = m->h;+ if (!output_size(m) || m->w != w || m->h != h) {+ if (sample) {+ IMFSample_Release(sample);+ }+ return -1; /* resized mid-stream: anim.c reopens at its size (a seek) */+ }+ }+ if (flags & (MF_SOURCE_READERF_ENDOFSTREAM | MF_SOURCE_READERF_ERROR)) {+ if (sample) {+ IMFSample_Release(sample);+ }+ return (flags & MF_SOURCE_READERF_ERROR) ? -1 : 0;+ }+ if (!sample) {+ continue; /* a stream tick (gap): read on */+ }+ LONGLONG d = 0;+ const bool has_dur = SUCCEEDED(IMFSample_GetSampleDuration(sample, &d)) && d > 0;+ const bool ok = copy_frame(m, sample, dst);+ IMFSample_Release(sample);+ if (!ok) {+ return -1;+ }+ *r_pts = (double)ts / 1e7;+ *r_dur = has_dur ? (double)d / 1e7 : m->frame_dur;+ return 1;+ }+ return -1;+}++static bool mf_seek(void *state, double t)+{+ MFMovie *m = (MFMovie *)state;+ PROPVARIANT pos;+ memset(&pos, 0, sizeof(pos));+ pos.vt = VT_I8;+ pos.hVal.QuadPart = (LONGLONG)(t * 1e7);+ static const GUID time_format = {0}; /* GUID_NULL: 100 ns units */+ return SUCCEEDED(IMFSourceReader_SetCurrentPosition(m->reader, &time_format, &pos));+}++static size_t mf_bytes(const void *state)+{+ const MFMovie *m = (const MFMovie *)state;+ /* what this process holds for it: the reader's output samples (an estimate;+ * decoder surfaces live in video memory) */+ return sizeof(*m) + (size_t)m->w * m->h * 3;+}++const BKImAnimCodec imb_movie_mf = {+ "mf", true, imb_is_a_movie, mf_open, mf_next, mf_seek, mf_bytes, mf_close};++/** \} */++#endif /* BUI_VIDEO_MF */diff --git a/kit/intern/image/play.c b/kit/intern/image/play.cindex 5d85688..d8ab3cb 100644--- a/kit/intern/image/play.c+++ b/kit/intern/image/play.c@@ -1,36 +1,59 @@-/* Playing an animation held by a cache entry: the frame clock.+/* The one clock of everything that plays: GIF, animated WebP and video,+ * owned by a cache entry (a shared <img>, or a <video>'s private entry).** The entry shows one frame; this keeps the decoder and the next frame,* decoded ahead on the workers (one step request at a time), and says when- * the next frame is due (each frame's own delay, GIF and WebP play counts).- * It only moves on when the frame on screen was drawn: a window that isn't- * rendering (hidden, minimized) costs one decode, then nothing, and an- * entry nobody shows is not stepped at all (cache.c).+ * the next frame is due (each frame's own duration over the playback rate,+ * GIF and WebP play counts, a video's loop). It only moves on when the frame+ * on screen was drawn: a window that isn't rendering (hidden, minimized)+ * costs one decode, then nothing, and an entry nobody shows is not stepped+ * at all (cache.c). Movies decode into the frame the screen no longer shows+ * (two frames, no allocation per frame).+ *+ * Controls (pause, seek, rate, loop) are the <video> element's; a seek's+ * frame shows as soon as it is decoded, paused or not.** SPDX-License-Identifier: GPL-2.0-or-later */+#include <float.h>+#include "MEM_guardedalloc.h"#include "image.h"struct BKImPlay {- BKImAnim *anim; /* NULL while a step request has it */+ BKImAnim *anim; /* NULL before the first frame and while a step request has it */BKImBuf *next; /* the frame after the one shown (NULL while decoding) */- int next_delay;+ double next_pts, next_dur;bool next_wrapped; /* `next` starts another play */- int delay; /* how long the shown frame stays (ms) */+ bool next_seek; /* `next` is a seek's frame: shown at once */+ BKImBuf *spare; /* movies: the frame the screen let go of, decoded into next */+ double pts, dur; /* the shown frame's time and duration (seconds) */double due; /* when it is over (bk_time seconds; 0: not timed yet) */- int plays, loops; /* plays finished; loops 0 = forever */- bool drawn; /* the shown frame reached the screen */- bool finished; /* the last play's last frame is showing */+ int plays; /* plays finished */+ int loops; /* the play count (0 = forever): the file's, or set */+ int loops_set; /* -1: the file's */+ int file_loops, src_w, src_h;+ double duration; /* seconds, 0 when unknown */+ double rate;+ double seek; /* a seek to request (< 0: none) */+ double seek_t; /* the last seek's time, while its frame isn't shown */+ bool seeking;+ unsigned seeks; /* seeks asked for; a step taken before the last one is stale */+ unsigned step_seeks; /* `seeks` when the step out was taken */+ bool step_seek; /* the step out seeks */+ bool loaded, failed, paused, drawn;+ bool finished; /* the last play's last frame shows */};-BKImPlay *imb_play_new(BKImAnim *anim, int delay)+BKImPlay *imb_play_new(bool paused, int loops){BKImPlay *p = (BKImPlay *)MEM_callocN(sizeof(BKImPlay), "BKImPlay");- p->anim = anim;- p->delay = delay;- p->loops = imb_anim_loops(anim);+ p->paused = paused;+ p->loops_set = loops;+ p->loops = BK_MAX(0, loops);+ p->rate = 1.0;+ p->seek = -1.0;return p;}@@ -39,77 +62,199 @@ void imb_play_free(BKImPlay *p)if (p) {imb_anim_free(p->anim);BK_imbuf_free(p->next);+ BK_imbuf_free(p->spare);MEM_freeN(p);}}size_t imb_play_bytes(const BKImPlay *p){- return sizeof(*p) + (p->anim ? imb_anim_bytes(p->anim) : 0) + (p->next ? p->next->size : 0);+ return sizeof(*p) + (p->anim ? imb_anim_bytes(p->anim) : 0) + (p->next ? p->next->size : 0) ++ (p->spare ? p->spare->size : 0);+}++void imb_play_start(BKImPlay *p, BKImAnim *anim, double pts, double dur)+{+ p->anim = anim;+ p->loaded = true;+ p->pts = pts;+ p->dur = dur;+ const BKImAnimInfo *info = imb_anim_info(anim);+ p->file_loops = info->loops;+ p->src_w = info->src_w;+ p->src_h = info->src_h;+ p->duration = info->duration;+ if (p->loops_set < 0) {+ p->loops = p->file_loops;+ }+}++void imb_play_fail(BKImPlay *p)+{+ p->failed = true;+}++void imb_play_recycle(BKImPlay *p, BKImBuf *old)+{+ if (old && old->stream && !p->spare) {+ p->spare = old; /* the next decode goes into it */+ }+ else {+ BK_imbuf_free(old);+ }}-BKImAnim *imb_play_take_step(BKImPlay *p)+BKImAnim *imb_play_take_step(BKImPlay *p, BKImBuf **r_reuse, double *r_seek){- if (!p->anim || p->next || p->finished) {+ if (!p->anim || p->failed || (p->seek < 0.0 && (p->next || p->finished))) {return NULL;}BKImAnim *a = p->anim;p->anim = NULL;+ *r_reuse = p->spare;+ p->spare = NULL;+ *r_seek = p->seek;+ p->step_seek = p->seek >= 0.0;+ p->step_seeks = p->seeks;+ p->seek = -1.0;return a;}-void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, int delay, bool wrapped)+void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, double pts, double dur,+ bool wrapped){p->anim = anim;- BK_imbuf_free(p->next);- p->next = next;- p->next_delay = delay;- p->next_wrapped = wrapped;+ if (p->step_seeks != p->seeks) {+ imb_play_recycle(p, next); /* decoded before a seek: stale */+ return;+ }if (!next) {p->finished = true; /* a decoding error: stay on this frame */+ return;}+ imb_play_recycle(p, p->next);+ p->next = next;+ p->next_pts = pts;+ p->next_dur = dur;+ p->next_wrapped = wrapped;+ p->next_seek = p->step_seek;}-void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim)+void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim, BKImBuf *reuse){p->anim = anim; /* stepped again when it is wanted */+ imb_play_recycle(p, reuse);+ if (p->step_seek && p->step_seeks == p->seeks && p->seek < 0.0) {+ p->seek = p->seek_t; /* the seek is still to do */+ }}void imb_play_drawn(BKImPlay *p, double now){p->drawn = true;if (p->due == 0.0) {- p->due = now + p->delay / 1000.0; /* the clock starts when it first shows */+ p->due = now + p->dur / p->rate; /* the clock starts when it first shows */}}-void imb_play_rewind_clock(BKImPlay *p)-{- p->due = 0.0; /* resumed: the frame gets its whole delay again */-}-double imb_play_due(const BKImPlay *p){- return p->drawn && !p->finished && p->next ? p->due : 0.0;+ if (p->next && p->next_seek) {+ return DBL_MIN; /* now */+ }+ return p->drawn && !p->finished && !p->paused && p->next ? p->due : 0.0;}BKImBuf *imb_play_advance(BKImPlay *p, double now){- if (imb_play_due(p) == 0.0 || now < p->due) {+ const double due = imb_play_due(p);+ if (due == 0.0 || now < due) {return NULL;}- if (p->next_wrapped && p->loops > 0 && p->plays + 1 >= p->loops) {+ if (!p->next_seek && p->next_wrapped && p->loops > 0 && p->plays + 1 >= p->loops) {p->finished = true; /* the last frame of the last play stays */- BK_imbuf_free(p->next);+ imb_play_recycle(p, p->next);p->next = NULL;return NULL;}BKImBuf *frame = p->next;p->next = NULL;- p->plays += p->next_wrapped;- p->delay = p->next_delay;- /* keep the beat, unless far behind (paused, slow decodes): restart it */- p->due = now - p->due > 0.25 ? now + p->delay / 1000.0 : p->due + p->delay / 1000.0;+ p->pts = p->next_pts;+ p->dur = p->next_dur;+ if (p->next_seek) {+ p->due = 0.0; /* timed from when it shows */+ p->next_seek = false;+ p->seeking = false;+ }+ else {+ p->plays += p->next_wrapped;+ /* keep the beat, unless far behind (paused, slow decodes): restart it */+ p->due = now - p->due > 0.25 ? now + p->dur / p->rate : p->due + p->dur / p->rate;+ }p->drawn = false;return frame;}++void imb_play_pause(BKImPlay *p, bool paused)+{+ if (p->paused && !paused) {+ p->due = 0.0; /* resumed: the frame gets its time again */+ if (p->finished) { /* playing an ended video starts it over */+ imb_play_seek(p, 0.0);+ }+ }+ p->paused = paused;+}++void imb_play_seek(BKImPlay *p, double t)+{+ const double duration = imb_play_duration(p);+ p->seek = p->seek_t = BK_CLAMP(t, 0.0, duration > 0.0 ? duration : t);+ p->seeking = true;+ p->seeks++;+ p->finished = false;+ p->plays = 0;+ imb_play_recycle(p, p->next); /* decoded before the seek */+ p->next = NULL;+ p->next_seek = false;+}++void imb_play_set_rate(BKImPlay *p, double rate)+{+ p->rate = BK_CLAMP(rate, 1.0 / 16.0, 16.0);+}++void imb_play_set_loops(BKImPlay *p, int loops)+{+ p->loops_set = loops;+ p->loops = loops >= 0 ? loops : p->file_loops;+ if (p->loops == 0) {+ p->finished = false; /* an ended video looping again goes on from the top */+ }+}++BKImPlayState imb_play_state(const BKImPlay *p)+{+ return p->failed ? BK_PLAY_ERROR :+ !p->loaded ? BK_PLAY_LOADING :+ p->finished ? BK_PLAY_ENDED :+ p->paused ? BK_PLAY_PAUSED :+ BK_PLAY_PLAYING;+}++double imb_play_time(const BKImPlay *p)+{+ return p->seeking ? p->seek_t : p->pts;+}++double imb_play_duration(const BKImPlay *p)+{+ return p->duration;+}++bool imb_play_size(const BKImPlay *p, int *r_w, int *r_h)+{+ *r_w = p->src_w;+ *r_h = p->src_h;+ return p->loaded;+}diff --git a/kit/intern/image/video.c b/kit/intern/image/video.cnew file mode 100644index 0000000..c8463a2--- /dev/null+++ b/kit/intern/image/video.c@@ -0,0 +1,361 @@+/* <video>: a private cache entry (cache.c: the same requests, workers and+ * visibility tracking as images) with its own clock (play.c: pause, seek,+ * rate, loop), shown on "video" and "image" nodes and canvas items, or+ * copied into a texture for Dear ImGui. The "video" node draws Blender-style+ * controls over it: a play/pause button and a scrubber with the time, like+ * the Timeline's playback controls and frame slider.+ *+ * SPDX-License-Identifier: GPL-2.0-or-later */++#include <math.h>+#include <stdio.h>+#include <string.h>++#include "MEM_guardedalloc.h"++#include "image.h"+#include "kit_intern.h"++struct BKVideo {+ BKImEntry *entry; /* frames and clock */+ BKImageRef poster; /* shown until the first frame */+ bool controls;+ bool scrubbing; /* the scrubber is dragged (the video was playing: `resume`) */+ bool resume;+ void (*fn)(BKVideo *v, int state, void *data);+ void *data;+ uint64_t tex_key; /* BK_video_texture: the frame last copied */+};++static BKImPlay *play_of(const BKVideo *v)+{+ return bk_image_entry_play(v->entry);+}++static void heard(void *owner)+{+ BKVideo *v = (BKVideo *)owner;+ if (v->fn) {+ v->fn(v, BK_video_state(v), v->data);+ }+}++BKVideo *BK_video_new(const char *src)+{+ BKImageSource *s = bk_image_source_get(src);+ if (!s) {+ return NULL;+ }+ BKVideo *v = (BKVideo *)MEM_callocN(sizeof(BKVideo), "BKVideo");+ /* muted autoplay, played once (HTML: loop off) */+ v->entry = bk_image_entry_new(s, imb_play_new(false, 1), heard, v);+ bk_image_source_release(s); /* the entry keeps it */+ return v;+}++void BK_video_free(BKVideo *v)+{+ if (v) {+ bk_image_entry_free(v->entry);+ bk_image_ref_clear(&v->poster);+ MEM_freeN(v);+ }+}++/* A control changed the clock: decode and redraw now, tell the listener. */+static void changed(BKVideo *v)+{+ bk_image_entry_changed(v->entry);+}++void BK_video_play(BKVideo *v)+{+ if (v) {+ imb_play_pause(play_of(v), false);+ changed(v);+ }+}++void BK_video_pause(BKVideo *v)+{+ if (v) {+ imb_play_pause(play_of(v), true);+ changed(v);+ }+}++void BK_video_seek(BKVideo *v, double seconds)+{+ if (v && isfinite(seconds)) {+ imb_play_seek(play_of(v), seconds);+ changed(v);+ }+}++void BK_video_set_rate(BKVideo *v, double rate)+{+ if (v && rate > 0.0 && isfinite(rate)) {+ imb_play_set_rate(play_of(v), rate);+ }+}++void BK_video_set_loop(BKVideo *v, int loop)+{+ if (v) {+ imb_play_set_loops(play_of(v), loop ? 0 : 1);+ changed(v);+ }+}++void BK_video_set_poster(BKVideo *v, const char *src)+{+ if (v) {+ if (src && src[0]) {+ bk_image_ref_set_source(&v->poster, src);+ }+ else {+ bk_image_ref_clear(&v->poster);+ }+ }+}++void BK_video_set_controls(BKVideo *v, int on)+{+ if (v) {+ v->controls = on != 0;+ changed(v);+ }+}++int BK_video_state(const BKVideo *v)+{+ return v ? (int)imb_play_state(play_of(v)) : BK_VIDEO_ERROR;+}++double BK_video_time(const BKVideo *v)+{+ return v ? imb_play_time(play_of(v)) : 0.0;+}++double BK_video_duration(const BKVideo *v)+{+ return v ? imb_play_duration(play_of(v)) : 0.0;+}++int BK_video_size(const BKVideo *v, int *r_w, int *r_h)+{+ int w = 0, h = 0;+ const bool ok = v && bk_video_size(v, &w, &h);+ if (r_w) {+ *r_w = w;+ }+ if (r_h) {+ *r_h = h;+ }+ return ok;+}++void BK_video_set_listener(BKVideo *v, void (*fn)(BKVideo *v, int state, void *data), void *data)+{+ if (v) {+ v->fn = fn;+ v->data = data;+ }+}++bool bk_video_size(const BKVideo *v, int *r_w, int *r_h)+{+ return imb_play_size(play_of(v), r_w, r_h) || bk_image_ref_size(&v->poster, r_w, r_h);+}++const BKImBuf *bk_video_frame(BKVideo *v, const BKImageFit *target, const int rect[4])+{+ const BKImBuf *ib = bk_image_entry_get(v->entry, target, rect);+ if (!ib && v->poster.kind == BK_IMREF_SOURCE) {+ ib = bk_image_cache_get(v->poster.u.source, target, rect);+ }+ return ib;+}++int BK_video_texture(BKVideo *v, int tex, int max_w, int max_h)+{+ if (!v) {+ return 0;+ }+ const BKImageFit target = imb_fit_within(max_w, max_h);+ const BKImBuf *ib = bk_image_entry_get(v->entry, &target, NULL); /* shown by the host */+ if (!ib || ib->format == BUI_PIXMAP_ARGB) {+ return ib && !tex ? BK_imbuf_texture(ib, 0) : tex; /* the poster, or nothing yet */+ }+ int tw = 0, th = 0;+ if (tex && (!BUI_texture_size(tex, &tw, &th) || tw != ib->x || th != ib->y)) {+ BUI_texture_free(tex); /* resized: a new texture */+ tex = 0;+ }+ if (!tex) {+ tex = BK_imbuf_texture(ib, 0);+ }+ else if (ib->key != v->tex_key) {+ BK_imbuf_texture_update(ib, 0, tex); /* in place */+ }+ v->tex_key = tex ? ib->key : 0;+ return tex;+}++/* -------------------------------------------------------------------- */+/** \name The "video" node: the frame, fitted, and Blender-style controls+ * \{ */++void BK_node_set_video(BKNode *n, BKVideo *v, int fit)+{+ bk_image_ref_set_video(&n->image, v);+ n->image_fit = fit;+ bk_relayout(n);+ bk_invalidate(n);+}++static BKVideo *node_video(const BKNode *n)+{+ return n->image.kind == BK_IMREF_VIDEO ? n->image.u.video : NULL;+}++/* The controls bar along the bottom: the play button, then the scrubber. */+static bool controls_rects(const BKNode *n, BKRect *r_button, BKRect *r_scrub)+{+ const BKVideo *v = node_video(n);+ const int uu = bk_unit(), pad = BK_MAX(2, uu / 5);+ if (!v || !v->controls || n->h < uu + 2 * pad || n->w < 3 * uu) {+ return false;+ }+ const int y = n->y + n->h - uu - pad;+ *r_button = bk_rect(n->x + pad, y, uu, uu);+ *r_scrub = bk_rect(n->x + 2 * pad + uu, y, n->w - 3 * pad - uu, uu);+ return true;+}++static void video_measure(BKNode *n, int *rw, int *rh)+{+ if (!bk_node_image_size(n, rw, rh)) { /* HTML's default, until the size is known */+ *rw = (int)(300 * bk_px());+ *rh = (int)(150 * bk_px());+ }+}++static void format_time(char *buf, size_t size, double t)+{+ const int s = (int)BK_MAX(0.0, t);+ snprintf(buf, size, "%d:%02d", s / 60, s % 60);+}++static void video_draw(BKNode *n)+{+ const float box[4] = {(float)n->x, (float)n->y, (float)n->w, (float)n->h};+ bk_image_ref_draw(&n->image, n->image_fit, 0, box, NODE_ENABLED(n) ? 1.0f : 0.5f);+ BKRect rb, rs;+ if (!controls_rects(n, &rb, &rs)) {+ return;+ }+ const BKVideo *v = node_video(n);+ const int state = BK_video_state(v);+ BKWCol w;+ bk_widget_state(&w, &bk_theme.regular, NODE_ENABLED(n) ? 0 : ST_DISABLED);+ bk_widget_but(&w, &rb, 0, CNR_ALL);+ const bool playing = state == BK_VIDEO_PLAYING || (v->scrubbing && v->resume);+ bk_icon(bk_icons_lookup(playing ? "PAUSE" : "PLAY"), rb.xmin + bk_rw(&rb) * 0.5f,+ rb.ymin + bk_rh(&rb) * 0.5f, w.text, 1.0f, 0);+ /* the scrubber: Blender's number slider, filled to the time */+ const double dur = BK_video_duration(v), t = BK_video_time(v);+ bk_widget_state(&w, &bk_theme.numslider, v->scrubbing ? ST_SELECT : 0);+ bk_widget_numslider(&w, &rs, 0, CNR_ALL, dur > 0.0 ? (float)BK_CLAMP(t / dur, 0.0, 1.0) : 0.0f);+ char a[16], b[16], label[40];+ format_time(a, sizeof(a), t);+ format_time(b, sizeof(b), dur);+ snprintf(label, sizeof(label), "%s / %s", a, b);+ bk_text_in(&rs, label, w.text, BK_ALIGN_CENTER, false);+}++static void scrub_to(BKNode *n, int x)+{+ BKRect rb, rs;+ BKVideo *v = node_video(n);+ const double dur = BK_video_duration(v);+ if (controls_rects(n, &rb, &rs) && dur > 0.0) {+ const double f = (x - rs.xmin) / (double)BK_MAX(1, bk_rw(&rs));+ BK_video_seek(v, BK_CLAMP(f, 0.0, 1.0) * dur);+ }+}++static int video_press(BKNode *n, BKEvent *e)+{+ BKRect rb, rs;+ BKVideo *v = node_video(n);+ if (!controls_rects(n, &rb, &rs) || !NODE_ENABLED(n)) {+ return 0;+ }+ if (bk_rect_inside(&rb, e->x, e->y)) {+ if (BK_video_state(v) == BK_VIDEO_PLAYING) {+ BK_video_pause(v);+ }+ else {+ BK_video_play(v);+ }+ return 1;+ }+ if (bk_rect_inside(&rs, e->x, e->y)) {+ /* scrubbing holds the video, like dragging Blender's frame slider */+ v->resume = BK_video_state(v) == BK_VIDEO_PLAYING;+ v->scrubbing = true;+ BK_video_pause(v);+ scrub_to(n, e->x);+ return 1;+ }+ return 0;+}++static int video_drag(BKNode *n, BKEvent *e)+{+ BKVideo *v = node_video(n);+ if (v && v->scrubbing) {+ scrub_to(n, e->x);+ }+ return 1;+}++static int video_release(BKNode *n, BKEvent *e)+{+ (void)e;+ BKVideo *v = node_video(n);+ if (v && v->scrubbing) {+ v->scrubbing = false;+ if (v->resume) {+ BK_video_play(v);+ }+ }+ return 1;+}++static int video_key(BKNode *n, BKEvent *e)+{+ BKVideo *v = node_video(n);+ if (!v || !v->controls || !bk_keysym_is(e->keysym, "space")) {+ return 0;+ }+ if (BK_video_state(v) == BK_VIDEO_PLAYING) {+ BK_video_pause(v);+ }+ else {+ BK_video_play(v);+ }+ return 1;+}++const BKClass bk_cls_video = {.name = "video",+ .flags = CLS_INTERACTIVE,+ .measure = video_measure,+ .draw = video_draw,+ .press = video_press,+ .drag = video_drag,+ .release = video_release,+ .key = video_key};++/** \} */diff --git a/kit/intern/kit_core.c b/kit/intern/kit_core.cindex c83176b..275af1f 100644--- a/kit/intern/kit_core.c+++ b/kit/intern/kit_core.c@@ -158,7 +158,7 @@ static const BKClass *const g_classes[] = {&bk_cls_menubutton, &bk_cls_menubar, &bk_cls_list, &bk_cls_tree,&bk_cls_gridview, &bk_cls_ramp, &bk_cls_curve, &bk_cls_normal,&bk_cls_histogram, &bk_cls_frame, &bk_cls_radio, &bk_cls_canvas,- &bk_cls_textarea, &bk_cls_image, &bk_cls_meter, &bk_cls_custom,+ &bk_cls_textarea, &bk_cls_image, &bk_cls_meter, &bk_cls_custom, &bk_cls_video,&bk_cls_filebrowser,};diff --git a/kit/intern/kit_frame.c b/kit/intern/kit_frame.cindex 600c8bd..abbf005 100644--- a/kit/intern/kit_frame.c+++ b/kit/intern/kit_frame.c@@ -824,10 +824,14 @@ static void citem_assign(CItem *c, const BKCanvasItem *item)memcpy(c->text, item->text, (size_t)tl);}/* one pixel source per image item (validated here, at the boundary) */- const int sources = (item->rgba != NULL) + (item->imbuf != NULL) + (item->src != NULL);+ const int sources = (item->rgba != NULL) + (item->imbuf != NULL) + (item->src != NULL) ++ (item->video != NULL);if (item->type != BK_ITEM_IMAGE || sources != 1) {bk_image_ref_clear(&c->img);}+ else if (item->video) {+ bk_image_ref_set_video(&c->img, item->video);+ }else if (item->imbuf) {bk_image_ref_set_imbuf(&c->img, item->imbuf);}@@ -843,6 +847,7 @@ static void citem_assign(CItem *c, const BKCanvasItem *item)c->it.rgba = NULL;c->it.src = NULL;c->it.imbuf = NULL;+ c->it.video = NULL;}/* An image item's box in canvas space: x0 y0 x1 y1 with 4 coordinates,diff --git a/kit/intern/kit_intern.h b/kit/intern/kit_intern.hindex b20b319..1a25136 100644--- a/kit/intern/kit_intern.h+++ b/kit/intern/kit_intern.h@@ -31,6 +31,11 @@ static inline BKRect bk_rect(int x, int y, int w, int h)BKRect r = {x, y, x + w, y + h};return r;}+/* BLI_rcti_isect_pt */+static inline bool bk_rect_inside(const BKRect *r, int x, int y)+{+ return x >= r->xmin && x < r->xmax && y >= r->ymin && y < r->ymax;+}/* Corner flags (Blender's CNR_*; "top" = screen top). */enum { CNR_TL = 1, CNR_TR = 2, CNR_BR = 4, CNR_BL = 8, CNR_ALL = 15 };@@ -551,6 +556,7 @@ void bk_theme_env_init(void);/* Custom nodes (kit_custom.c). The host timer that wakes them. */#define BK_TIMER_CUSTOM (-1)extern const BKClass bk_cls_custom;+extern const BKClass bk_cls_video; /* image/video.c */#define BK_IS_CUSTOM(n) ((n) && (n)->cls == &bk_cls_custom)/* Delivers an event (fills x, y relative to the node); returns its result. */int bk_custom_event(BKNode *n, BKCustomEvent *e, int wx, int wy);diff --git a/tests/video_test.c b/tests/video_test.cnew file mode 100644index 0000000..9129385--- /dev/null+++ b/tests/video_test.c@@ -0,0 +1,353 @@+/* Video: load, play, pause, seek, rate, loop and end through the public API+ * (BK_video_*), drawn on a canvas like an app would; nothing decoded while+ * off screen; malformed files; no allocation per frame once playing; every+ * MEM block back after BK_video_free + BK_image_exit. Exits 77 (skipped)+ * when the build or the machine has no video backend that reads the clip.+ *+ * SPDX-License-Identifier: GPL-2.0-or-later */+#include <math.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#ifdef _WIN32+# define WIN32_LEAN_AND_MEAN+# include <windows.h>+# define sleep_ms(ms) Sleep(ms)+#else+# include <time.h>+# include <unistd.h>+# define sleep_ms(ms) usleep((ms) * 1000)+#endif++#include "BUI_kit.h"+#include "GPU_raster.h"+#include "MEM_guardedalloc.h"++static int g_fail = 0;+#define CHECK(c) \+ do { \+ if (!(c)) { \+ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); \+ g_fail = 1; \+ } \+ } while (0)++static double now_s(void)+{+#ifdef _WIN32+ LARGE_INTEGER f, c;+ QueryPerformanceFrequency(&f);+ QueryPerformanceCounter(&c);+ return (double)c.QuadPart / (double)f.QuadPart;+#else+ struct timespec ts;+ clock_gettime(CLOCK_MONOTONIC, &ts);+ return ts.tv_sec + ts.tv_nsec / 1e9;+#endif+}++/* A host whose one timer the test loop fires. */+static struct {+ double due;+ int timer, requests;+} g_host;++static void host_schedule(void *data, int ms, int timer)+{+ (void)data;+ g_host.due = now_s() + ms / 1000.0;+ g_host.timer = timer;+}++static void host_request(void *data)+{+ (void)data;+ g_host.requests++;+}++#define W 200+#define H 120++/* Runs the context like a window loop for `seconds` (or until `until`+ * returns nonzero); returns the frames rendered. */+static int run(BKContext *ctx, BuiRaster *r, double seconds, int (*until)(void *), void *arg)+{+ const double end = now_s() + seconds;+ int frames = 0;+ while (now_s() < end && !(until && until(arg))) {+ if (g_host.due > 0.0 && now_s() >= g_host.due) {+ g_host.due = 0.0;+ BK_timer(ctx, g_host.timer);+ }+ BK_image_poll();+ if (BK_needs_render(ctx)) {+ frames += BK_render(ctx, r, W, H, 0) > 0;+ }+ sleep_ms(1);+ }+ return frames;+}++static int loaded(void *v)+{+ return BK_video_state((BKVideo *)v) != BK_VIDEO_LOADING;+}++static int ended(void *v)+{+ return BK_video_state((BKVideo *)v) == BK_VIDEO_ENDED;+}++static uint64_t box_hash(const uint32_t *px)+{+ uint64_t h = 1469598103934665603ull;+ for (int y = 10; y < 64; y++) {+ for (int x = 10; x < 106; x++) {+ h = (h ^ (px[y * W + x] & 0xFFFFFF)) * 1099511628211ull;+ }+ }+ return h;+}++static int g_events[8];+static void listener(BKVideo *v, int state, void *data)+{+ (void)v;+ (void)data;+ g_events[state & 7]++;+}++static void show(BKNode *canvas, BKVideo *v)+{+ static const float box[4] = {10, 10, 106, 64}; /* the clip's 96 x 54 */+ BKCanvasItem it;+ memset(&it, 0, sizeof(it));+ it.id = 1;+ it.type = BK_ITEM_IMAGE;+ it.coords = box;+ it.ncoords = 4;+ it.video = v;+ it.fit = BK_FIT_FILL;+ BK_canvas_set_item(canvas, &it);+}++static void test_playback(const char *clip)+{+ BKNode *canvas = BK_node_new("canvas");+ BKContext *ctx = BK_context_new(canvas);+ BKHost host = {0};+ host.request = host_request;+ host.schedule = host_schedule;+ BK_context_set_host(ctx, &host);+ BuiRaster *r = BUI_raster_create(W, H);+ const uint32_t *px = BUI_raster_pixels(r);++ /* loads when first drawn, then plays (muted autoplay) */+ BKVideo *v = BK_video_new(clip);+ BK_video_set_listener(v, listener, NULL);+ show(canvas, v);+ const double t0 = now_s();+ BK_render(ctx, r, W, H, 1);+ run(ctx, r, 5.0, loaded, v);+ printf("first frame after %.1f ms; backends: %s\n", (now_s() - t0) * 1000.0,+ BK_video_backends());+ int w = 0, h = 0;+ CHECK(BK_video_state(v) == BK_VIDEO_PLAYING);+ CHECK(BK_video_size(v, &w, &h) && w == 96 && h == 54);+ CHECK(fabs(BK_video_duration(v) - 2.0) < 0.15);+ CHECK(g_events[BK_VIDEO_PLAYING] == 1);++ /* it plays: the picture moves and the time goes on */+ uint64_t last = box_hash(px);+ int changes = 0;+ for (int i = 0; i < 10; i++) {+ run(ctx, r, 0.06, NULL, NULL);+ const uint64_t c = box_hash(px);+ changes += c != last;+ last = c;+ }+ const double t_play = BK_video_time(v);+ printf("playing 0.6 s: %d frame changes, at %.2f s\n", changes, t_play);+ CHECK(changes >= 3 && t_play > 0.2);++ /* paused: nothing moves */+ BK_video_pause(v);+ run(ctx, r, 0.1, NULL, NULL);+ const uint64_t held = box_hash(px);+ const double t_held = BK_video_time(v);+ run(ctx, r, 0.3, NULL, NULL);+ CHECK(box_hash(px) == held && BK_video_time(v) == t_held);+ CHECK(BK_video_state(v) == BK_VIDEO_PAUSED);++ /* seek while paused: that frame shows */+ BK_video_seek(v, 1.5);+ run(ctx, r, 0.4, NULL, NULL);+ printf("seek to 1.5 s: at %.3f s\n", BK_video_time(v));+ CHECK(fabs(BK_video_time(v) - 1.5) < 0.11);+ CHECK(box_hash(px) != held);++ /* no allocation per frame once playing (MEM: requests and frames recycle) */+ BK_video_seek(v, 0.0);+ BK_video_play(v);+ run(ctx, r, 0.3, NULL, NULL);+ const size_t allocs = MEM_get_allocations_total();+ const unsigned blocks = MEM_get_memory_blocks_in_use();+ const double ta = BK_video_time(v);+ run(ctx, r, 0.6, NULL, NULL);+ const double frames_played = (BK_video_time(v) - ta) * 10.0;+ printf("playing %.0f frames: %zu MEM allocations, %d blocks change\n", frames_played,+ MEM_get_allocations_total() - allocs, (int)MEM_get_memory_blocks_in_use() - (int)blocks);+ CHECK(MEM_get_allocations_total() == allocs);++ /* rate and end: 4x from 1 s plays the rest in 0.25 s, then it ends (once) */+ BK_video_seek(v, 1.0);+ BK_video_set_rate(v, 4.0);+ const double te = now_s();+ run(ctx, r, 3.0, ended, v);+ printf("4x from 1 s ended after %.2f s\n", now_s() - te);+ CHECK(BK_video_state(v) == BK_VIDEO_ENDED && g_events[BK_VIDEO_ENDED] == 1);+ CHECK(now_s() - te < 1.0);++ /* loop: it goes round */+ BK_video_set_loop(v, 1);+ BK_video_play(v);+ double lowest = 10.0, highest = 0.0;+ for (int i = 0; i < 40; i++) {+ run(ctx, r, 0.03, NULL, NULL);+ const double t = BK_video_time(v);+ lowest = t < lowest ? t : lowest;+ highest = t > highest ? t : highest;+ }+ printf("looping 1.2 s at 4x: time between %.2f and %.2f\n", lowest, highest);+ CHECK(BK_video_state(v) == BK_VIDEO_PLAYING && lowest < 0.5 && highest > 1.5);++ /* off screen: no decoding, no timers, no renders */+ BK_canvas_delete(canvas, -1);+ BK_render(ctx, r, W, H, 1);+ BKImageStats st;+ for (int i = 0; i < 200; i++) {+ run(ctx, r, 0.01, NULL, NULL);+ BK_image_stats(&st);+ if (st.pending == 0) {+ break;+ }+ }+ run(ctx, r, 0.05, NULL, NULL);+ BK_image_stats(&st);+ const int decoded = st.decoded, requests = g_host.requests;+ const double tv = BK_video_time(v);+ const int frames = run(ctx, r, 0.5, NULL, NULL);+ BK_image_stats(&st);+ printf("off screen for 0.5 s: %d decodes, %d renders, %d render requests\n",+ st.decoded - decoded, frames, g_host.requests - requests);+ CHECK(st.decoded == decoded && frames == 0 && g_host.requests == requests);+ CHECK(BK_video_time(v) == tv);++ /* shown again: it resumes */+ show(canvas, v);+ BK_render(ctx, r, W, H, 1);+ run(ctx, r, 0.3, NULL, NULL);+ CHECK(BK_video_time(v) != tv);++ BK_canvas_delete(canvas, -1);+ BK_video_free(v);+ BK_context_free(ctx);+ BK_node_free(canvas);+ BUI_raster_free(r);+}++/* Truncated and bit-flipped copies: an error state or frames, never a crash. */+static void test_malformed(const char *clip)+{+ FILE *f = fopen(clip, "rb");+ if (!f) {+ CHECK(0);+ return;+ }+ static unsigned char data[1 << 16];+ const size_t n = fread(data, 1, sizeof(data), f);+ fclose(f);+ BKNode *canvas = BK_node_new("canvas");+ BKContext *ctx = BK_context_new(canvas);+ BuiRaster *r = BUI_raster_create(W, H);+ char path[600];+ snprintf(path, sizeof(path), "%s/bad.mp4", BUI_TEST_TMP);+ int errors = 0, played = 0;+ srand(7);+ for (int it = 0; it < 24; it++) {+ static unsigned char bad[1 << 16];+ memcpy(bad, data, n);+ size_t len = n;+ if (it % 3 == 0) {+ len = (size_t)rand() % n; /* truncated */+ }+ else {+ for (int k = 0; k < 1 + it; k++) { /* bytes flipped */+ bad[(size_t)rand() % n] ^= (unsigned char)(1 + rand() % 255);+ }+ }+ FILE *o = fopen(path, "wb");+ fwrite(bad, 1, len, o);+ fclose(o);+ BKVideo *v = BK_video_new(path);+ show(canvas, v);+ BK_render(ctx, r, W, H, 1);+ run(ctx, r, 2.0, loaded, v);+ run(ctx, r, 0.2, NULL, NULL); /* a few frames of whatever decodes */+ errors += BK_video_state(v) == BK_VIDEO_ERROR;+ played += BK_video_state(v) == BK_VIDEO_PLAYING || BK_video_state(v) == BK_VIDEO_ENDED;+ BK_canvas_delete(canvas, -1);+ BK_render(ctx, r, W, H, 1);+ BK_video_free(v);+ }+ remove(path);+ printf("24 malformed clips: %d errors, %d played, no crash\n", errors, played);+ CHECK(errors + played > 0);+ BK_context_free(ctx);+ BK_node_free(canvas);+ BUI_raster_free(r);+}++int main(void)+{+ setvbuf(stdout, NULL, _IONBF, 0);+ if (!BK_video_backends()[0]) {+ printf("video_test: no video backend in this build, skipped\n");+ return 77;+ }+ const unsigned blocks0 = MEM_get_memory_blocks_in_use();+ const size_t bytes0 = MEM_get_memory_in_use();+ char clip[600];+ snprintf(clip, sizeof(clip), "%s/%s", BUI_TEST_VIDEOS, getenv("BUI_TEST_CLIP") ? getenv("BUI_TEST_CLIP") : "clip.mp4");+ {+ /* the machine may lack the decoder (Windows N without the Media Feature Pack) */+ BKVideo *probe = BK_video_new(clip);+ BKNode *canvas = BK_node_new("canvas");+ BKContext *ctx = BK_context_new(canvas);+ BuiRaster *r = BUI_raster_create(W, H);+ show(canvas, probe);+ BK_render(ctx, r, W, H, 1);+ run(ctx, r, 5.0, loaded, probe);+ const int state = BK_video_state(probe);+ BK_canvas_delete(canvas, -1);+ BK_video_free(probe);+ BK_context_free(ctx);+ BK_node_free(canvas);+ BUI_raster_free(r);+ BK_image_exit();+ if (state == BK_VIDEO_ERROR || state == BK_VIDEO_LOADING) {+ printf("video_test: %s does not decode %s here, skipped\n", BK_video_backends(), clip);+ return 77;+ }+ }+ memset(g_events, 0, sizeof(g_events));+ test_playback(clip);+ test_malformed(clip);+ BK_image_exit();+ printf("memory: %u blocks, %zu bytes before; %u blocks, %zu bytes after\n", blocks0, bytes0,+ MEM_get_memory_blocks_in_use(), MEM_get_memory_in_use());+ CHECK(MEM_get_memory_blocks_in_use() == blocks0);+ CHECK(MEM_get_memory_in_use() == bytes0);+ printf(g_fail ? "video_test: FAILED\n" : "video_test: ok\n");+ return g_fail;+}diff --git a/tests/videos/clip.mp4 b/tests/videos/clip.mp4new file mode 100644index 0000000..f46b45bBinary files /dev/null and b/tests/videos/clip.mp4 differdiff --git a/tests/videos/clip_av1.mp4 b/tests/videos/clip_av1.mp4new file mode 100644index 0000000..81f9feaBinary files /dev/null and b/tests/videos/clip_av1.mp4 differ