Skip to content

armgit / BUI

Public Imported from GitHub
0

Blender's GPU Accelerated UI, Decoupled from it's original codebase. Supports Rust, Python, JS, React, C & C++. Twice as good as GPUI

video decoder

Preetham Pemmasani74b99399cb61
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e3d9cb4..278d6bc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -99,6 +99,15 @@ if(BUI_BUILD_TESTS)
BUI_TEST_TMP="${CMAKE_CURRENT_BINARY_DIR}/image_test_files")
add_test(NAME bui_image_test COMMAND bui_image_test)
set_tests_properties(bui_image_test PROPERTIES TIMEOUT 120)
+ # video: play, pause, seek, rate, loop, off screen, malformed files, no leaks
+ # (exits 77, skipped, without a backend that decodes the H.264 clip)
+ add_executable(bui_video_test tests/video_test.c)
+ target_link_libraries(bui_video_test PRIVATE bui::kit)
+ target_compile_definitions(bui_video_test PRIVATE
+ BUI_TEST_VIDEOS="${CMAKE_CURRENT_SOURCE_DIR}/tests/videos"
+ BUI_TEST_TMP="${CMAKE_CURRENT_BINARY_DIR}")
+ add_test(NAME bui_video_test COMMAND bui_video_test)
+ set_tests_properties(bui_video_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 180)
if(WIN32) # the thumbnail-grid benchmark (bench/images/image_grid.c; not a test)
add_executable(bui_image_bench bench/images/image_grid.c)
target_link_libraries(bui_image_bench PRIVATE bui::kit psapi)
diff --git a/gpu/include/GPU_raster.h b/gpu/include/GPU_raster.h
index 289c8a7..bf310c4 100644
--- a/gpu/include/GPU_raster.h
+++ b/gpu/include/GPU_raster.h
@@ -146,6 +146,9 @@ enum {
enum {
BUI_IMAGE_NEAREST = 1 << 0, /* nearest texel (CSS image-rendering: pixelated); else bilinear */
};
+/* Rows y0 .. y0 + rows of `img` as premultiplied 0xAARRGGBB (`dst`: w per
+ * row): a copy, or NV12 converted the way drawing converts it. */
+void BUI_pixmap_read(const BuiPixmap *img, int y0, int rows, uint32_t *dst);
/* Draws the `src` part of `img` (x, y, w, h in image pixels; NULL = all of
* it) into x, y, w, h, filtered, times `alpha`, premultiplied source-over.
* 1:1 on whole pixels is an exact copy. */
diff --git a/gpu/intern/gpu_raster.c b/gpu/intern/gpu_raster.c
index fb170a1..1a91077 100644
--- a/gpu/intern/gpu_raster.c
+++ b/gpu/intern/gpu_raster.c
@@ -1300,6 +1300,23 @@ static inline uint32_t yuv_argb(const int *k, int y, int u, int v)
return 0xFF000000u | (uint32_t)r << 16 | (uint32_t)g << 8 | (uint32_t)b;
}
+void BUI_pixmap_read(const BuiPixmap *img, int y0, int rows, uint32_t *dst)
+{
+ if (img->format == BUI_PIXMAP_ARGB) {
+ memcpy(dst, img->px + (size_t)y0 * img->w, sizeof(uint32_t) * (size_t)img->w * rows);
+ return;
+ }
+ const int *k = k_yuv[img->format == BUI_PIXMAP_NV12_BT709];
+ const uint8_t *yp = (const uint8_t *)img->px, *uvp = yp + (size_t)img->w * img->h;
+ for (int y = y0; y < y0 + rows; y++) {
+ const uint8_t *yr = yp + (size_t)y * img->w, *cr = uvp + (size_t)(y >> 1) * img->w;
+ for (int x = 0; x < img->w; x++) {
+ const uint8_t *c = cr + (x & ~1);
+ *dst++ = yuv_argb(k, yr[x], c[0], c[1]);
+ }
+ }
+}
+
/* 8-bit bilinear of four samples (fx, fy: 0..256). */
static inline int lerp2(int a, int b, int c, int d, int fx, int fy)
{
diff --git a/kit/CMakeLists.txt b/kit/CMakeLists.txt
index d764dc0..11e8e46 100644
--- a/kit/CMakeLists.txt
+++ b/kit/CMakeLists.txt
@@ -44,6 +44,9 @@ add_library(bui_kit STATIC
intern/image/cache.c
intern/image/loader.c
intern/image/draw.c
+ intern/image/video.c
+ intern/image/movie.c
+ intern/image/movie_mf.c
# Blender's UI sources copied and edited for the kit (C++; see blender/bk_blender.h)
intern/blender/interface_string_search.cc
intern/blender/interface_utils.cc
@@ -62,6 +65,9 @@ option(BUI_IMAGE_SVG "SVG images through the kit's nanosvg (no extra code: the i
if(BUI_IMAGE_SVG)
target_compile_definitions(bui_kit PRIVATE BUI_IMAGE_SVG)
endif()
+if(WIN32)
+ target_compile_definitions(bui_kit PRIVATE BUI_VIDEO_MF)
+endif()
set(_bui_quiet "$<IF:$<C_COMPILER_ID:MSVC>,/w,-w>")
# Wuffs: one translation unit (format_wuffs.c includes wuffs-v0.4.c)
target_include_directories(bui_kit PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/extern/wuffs)
diff --git a/kit/include/BUI_kit.h b/kit/include/BUI_kit.h
index a374cc3..9b3437b 100644
--- a/kit/include/BUI_kit.h
+++ b/kit/include/BUI_kit.h
@@ -260,6 +260,7 @@ enum {
BK_IMAGE_HDR,
BK_IMAGE_PNM,
BK_IMAGE_SVG,
+ BK_IMAGE_VIDEO, /* a frame of a BKVideo */
};
/* CSS object-fit */
enum { BK_FIT_FILL = 0, BK_FIT_CONTAIN, BK_FIT_COVER, BK_FIT_NONE, BK_FIT_SCALE_DOWN };
@@ -303,6 +304,48 @@ BKImBuf *BK_anim_next(BKAnim *anim, int *r_delay_ms);
int BK_anim_info(const BKAnim *anim, int *r_w, int *r_h, int *r_loops);
void BK_anim_free(BKAnim *anim);
+/* -- video (kit/intern/image: one player for GIF, WebP and video) ---------------------
+ * A <video>: decoded by the OS (Media Foundation, AVFoundation, GStreamer) or
+ * an optional built-in decoder (dav1d, FFmpeg), streamed from the file at
+ * the size it is drawn, frames in one GPU texture updated in place (NV12:
+ * YCbCr converted in the shader; the CPU raster converts what it draws).
+ * It loads when first drawn and plays only while it shows: off screen,
+ * clipped, in a hidden window, nothing is decoded. Muted (no audio yet).
+ * UI-thread API; the state listener runs on the UI thread. */
+typedef struct BKVideo BKVideo;
+enum { BK_VIDEO_LOADING = 0, BK_VIDEO_PLAYING, BK_VIDEO_PAUSED, BK_VIDEO_ENDED, BK_VIDEO_ERROR };
+/* Autoplays (a muted autoplay, as browsers allow); BK_video_pause before the
+ * first draw shows the first frame instead. NULL for an empty src. */
+BKVideo *BK_video_new(const char *src);
+void BK_video_free(BKVideo *v);
+void BK_video_play(BKVideo *v); /* an ended video starts over */
+void BK_video_pause(BKVideo *v);
+void BK_video_seek(BKVideo *v, double seconds); /* the frame there shows, paused or not */
+void BK_video_set_rate(BKVideo *v, double rate); /* playback speed, 1/16 .. 16 */
+void BK_video_set_loop(BKVideo *v, int loop);
+/* An image file shown until the first frame (HTML poster); NULL: none. */
+void BK_video_set_poster(BKVideo *v, const char *src);
+/* Blender-style controls over it (play/pause, scrubber, time) in "video" nodes. */
+void BK_video_set_controls(BKVideo *v, int on);
+int BK_video_state(const BKVideo *v); /* BK_VIDEO_* */
+double BK_video_time(const BKVideo *v); /* seconds */
+double BK_video_duration(const BKVideo *v); /* seconds; 0 until loaded or unknown */
+int BK_video_size(const BKVideo *v, int *r_w, int *r_h); /* intrinsic; 0 until loaded */
+/* Called on every state change (loaded, playing, paused, ended, error). */
+void BK_video_set_listener(BKVideo *v, void (*fn)(BKVideo *v, int state, void *data), void *data);
+/* Dear ImGui and textured meshes: the current frame in a BUI_texture (RGBA),
+ * created when `tex` is 0 (or the size changed: the old one is freed) and
+ * updated in place when a new frame came; returns the texture (0 before
+ * the first frame). Calling it each frame is what keeps it playing: the
+ * host keeps rendering while BK_video_state is BK_VIDEO_PLAYING. */
+int BK_video_texture(BKVideo *v, int tex, int max_w, int max_h);
+/* Shown on a "video" node (fitted, with its controls) or an "image" node; the
+ * node borrows it (free the video after the node lets go: set NULL). */
+void BK_node_set_video(BKNode *node, BKVideo *v, int fit);
+/* The backends this build reads video with ("mf", "avfoundation", "gstreamer",
+ * "dav1d", "ffmpeg"), comma-separated; "" when built without video. */
+const char *BK_video_backends(void);
+
/* The shared cache and loader. Sources (file paths) decode on a few worker
* threads at the size they are drawn, visible first; results arrive on the UI
* thread (the window loop wakes for them) and redraw only where they show. */
@@ -355,6 +398,7 @@ typedef struct BKCanvasItem {
int fit; /* BK_FIT_* */
int frame; /* animated src: 0 plays (each frame for its delay, looping as the file
* says; paused while off screen); n > 0 shows frame n - 1 */
+ BKVideo *video; /* images: a video's current frame (borrowed, as imbuf) */
} BKCanvasItem;
void BK_canvas_set_item(BKNode *canvas, const BKCanvasItem *item); /* insert or replace by id */
void BK_canvas_add_item(BKNode *canvas, const BKCanvasItem *item);
diff --git a/kit/intern/image/anim.c b/kit/intern/image/anim.c
index 34922cf..37302db 100644
--- a/kit/intern/image/anim.c
+++ b/kit/intern/image/anim.c
@@ -1,12 +1,15 @@
-/* Animations (GIF, animated WebP) decoded one frame at a time, at the size
- * they are drawn: the cache keeps the frame on screen and the next one,
- * not the whole animation. Each format provides a BKImAnimCodec that
- * composites its frames (GIF disposal and blending, WebP's canvas) into a
- * full-size premultiplied frame; this file fits frames to the target (the
- * same plan as stills, imb_plan / imb_finish) and loops.
+/* Animated media, one frame at a time, at the size drawn: GIF, animated
+ * WebP and video behind one codec interface (BKImAnimCodec: Blender's
+ * MovieReader shape, open / next / seek / close). In-memory codecs composite
+ * frames (GIF disposal and blending, WebP's canvas) into a full-size
+ * premultiplied canvas, fitted here with the plan of stills (imb_plan /
+ * imb_finish); streamed codecs (movie.c: the OS decoders, dav1d, FFmpeg)
+ * read the file as they go and decode NV12 at about the size drawn into a
+ * recycled frame (no allocation per frame).
*
* SPDX-License-Identifier: GPL-2.0-or-later */
+#include <math.h>
#include <string.h>
#include "MEM_guardedalloc.h"
@@ -16,12 +19,17 @@
struct BKImAnim {
const BKImAnimCodec *codec;
void *state;
- uint8_t *file; /* owned copy of the encoded file (the codecs read from it) */
+ uint8_t *file; /* in-memory codecs: the encoded file they read (owned) */
size_t size;
- int w, h; /* canvas (intrinsic) size */
- int frames, loops;
+ char *path; /* streamed codecs: the file (owned) */
+ BKImAnimInfo info;
+ int frames; /* from the header, at least what was decoded */
int ftype; /* BK_IMAGE_* */
int index; /* the frame next() returns next */
+ double pts; /* its time (seconds) when the codec doesn't say */
+ double seek; /* the frame to return next shows at this time (< 0: none) */
+ uint64_t stream; /* streamed: the frames' BuiPixmap stream (one GPU texture) */
+ BKImageFit target;
BKImPlan plan; /* how frames are fitted */
};
@@ -36,31 +44,97 @@ const BKImAnimCodec *imb_anim_codec(const uint8_t *mem, size_t size)
return NULL;
}
+static BKImAnim *anim_new(const BKImAnimCodec *codec, const BKImageFit *target)
+{
+ BKImAnim *a = (BKImAnim *)MEM_callocN(sizeof(BKImAnim), "BKImAnim");
+ a->codec = codec;
+ a->target = *target;
+ a->seek = -1.0;
+ return a;
+}
+
BKImAnim *imb_anim_open(uint8_t *file, size_t size, const BKImageFit *target)
{
BKImageInfo info;
const BKImFileType *t = imb_file_type(file, size);
const BKImAnimCodec *codec = NULL;
- BKImAnim *a = NULL;
if (t && t->load_header(file, size, &info) && info.width > 0 && info.height > 0) {
codec = imb_anim_codec(file, size);
}
- if (codec) {
- a = (BKImAnim *)MEM_callocN(sizeof(BKImAnim), "BKImAnim");
- a->codec = codec;
- a->ftype = t->ftype;
- a->file = file;
- a->size = size;
- a->state = codec->open(file, size, &a->w, &a->h, &a->loops);
- a->frames = BK_MAX(1, info.frames);
- if (!a->state || !imb_plan(target, a->w, a->h, 1, &a->plan)) {
- imb_anim_free(a); /* frees `file` too */
- return NULL;
+ if (!codec) {
+ MEM_freeN(file);
+ return NULL;
+ }
+ BKImAnim *a = anim_new(codec, target);
+ a->ftype = t->ftype;
+ a->file = file;
+ a->size = size;
+ a->frames = BK_MAX(1, info.frames);
+ a->state = codec->open(NULL, file, size, 0, 0, &a->info);
+ if (!a->state || !imb_plan(target, a->info.w, a->info.h, 1, &a->plan)) {
+ imb_anim_free(a); /* frees `file` too */
+ return NULL;
+ }
+ return a;
+}
+
+/* Opens `a`'s streamed codec at its target (the plan's whole-frame size). */
+static bool movie_start(BKImAnim *a)
+{
+ BKImPlan plan;
+ if (a->info.src_w > 0 && !imb_plan(&a->target, a->info.src_w, a->info.src_h, 1, &plan)) {
+ return false; /* nothing of it shows */
+ }
+ const int want_w = a->info.src_w > 0 ? plan.fw : a->target.box_w;
+ const int want_h = a->info.src_h > 0 ? plan.fh : a->target.box_h;
+ BKImAnimInfo info;
+ memset(&info, 0, sizeof(info));
+ void *state = a->codec->open(a->path, NULL, 0, want_w, want_h, &info);
+ if (!state) {
+ return false;
+ }
+ if (info.w <= 0 || info.h <= 0 || (info.w | info.h) & 1 || info.w > BK_IMB_MAX_SIDE ||
+ info.h > BK_IMB_MAX_SIDE || info.src_w <= 0 || info.src_h <= 0 ||
+ !imb_plan(&a->target, info.src_w, info.src_h, 1, &plan))
+ {
+ a->codec->close(state); /* validated at the codec boundary: frames are even-sized NV12 */
+ return false;
+ }
+ if (a->state) {
+ a->codec->close(a->state);
+ }
+ a->state = state;
+ a->info = info;
+ a->plan = plan;
+ return true;
+}
+
+BKImAnim *imb_movie_open(const char *path, const BKImageFit *target)
+{
+ size_t n;
+ uint8_t *head = imb_read_file(path, 64, &n);
+ if (!head) {
+ return NULL;
+ }
+ BKImAnim *a = NULL;
+ for (int i = 0; imb_movie_codecs[i] && !a; i++) {
+ const BKImAnimCodec *codec = imb_movie_codecs[i];
+ if (codec->is_a && !codec->is_a(head, n)) {
+ continue;
+ }
+ a = anim_new(codec, target);
+ const size_t len = strlen(path);
+ a->path = (char *)MEM_mallocN(len + 1, "movie path");
+ memcpy(a->path, path, len + 1);
+ a->ftype = BK_IMAGE_VIDEO;
+ a->stream = imb_new_key();
+ if (!movie_start(a)) {
+ imb_anim_free(a);
+ a = NULL; /* the next backend may read it */
}
- return a;
}
- MEM_freeN(file);
- return NULL;
+ MEM_freeN(head);
+ return a;
}
void imb_anim_free(BKImAnim *a)
@@ -69,14 +143,18 @@ void imb_anim_free(BKImAnim *a)
if (a->state) {
a->codec->close(a->state);
}
+ if (a->stream) {
+ BUI_image_forget(a->stream); /* its GPU texture */
+ }
MEM_freeN(a->file);
+ MEM_freeN(a->path);
MEM_freeN(a);
}
}
-int imb_anim_loops(const BKImAnim *a)
+const BKImAnimInfo *imb_anim_info(const BKImAnim *a)
{
- return a->loops;
+ return &a->info;
}
int imb_anim_frames(const BKImAnim *a)
@@ -89,39 +167,124 @@ size_t imb_anim_bytes(const BKImAnim *a)
return sizeof(*a) + a->size + a->codec->bytes(a->state);
}
-/* The next composited frame at full size (a fresh buffer), wrapping to the
- * first after the last. NULL on a decoding error. */
-static BKImBuf *next_full(BKImAnim *a, int *r_delay, bool *r_wrapped)
+bool imb_anim_retarget(BKImAnim *a, const BKImageFit *target)
{
- BKImBuf *ib = imb_alloc(a->w, a->h, 1);
- if (!ib) {
- return NULL;
+ if (memcmp(&a->target, target, sizeof(*target)) == 0) {
+ return true;
}
- ib->ftype = a->ftype;
+ const BKImageFit old = a->target;
+ a->target = *target;
+ if (!a->codec->streamed) {
+ BKImPlan plan;
+ if (imb_plan(target, a->info.w, a->info.h, 1, &plan)) {
+ a->plan = plan;
+ return true;
+ }
+ }
+ else if (movie_start(a)) {
+ imb_anim_seek(a, a->pts); /* where it was */
+ return true;
+ }
+ a->target = old;
+ return false;
+}
+
+void imb_anim_seek(BKImAnim *a, double t)
+{
+ a->seek = BK_MAX(0.0, t);
+}
+
+/* The next frame from the codec into `dst`, wrapping to the first after the
+ * last; -1 on errors. */
+static int codec_next(BKImAnim *a, BKImBuf *dst, double *r_pts, double *r_dur, bool *r_wrapped)
+{
*r_wrapped = false;
- int got = a->codec->next(a->state, ib->rect, r_delay);
- if (got == 0 && a->index > 0) { /* past the last frame: from the top */
- a->codec->rewind(a->state);
+ if (a->seek >= 0.0) {
+ a->codec->seek(a->state, a->seek);
a->index = 0;
- *r_wrapped = true;
- got = a->codec->next(a->state, ib->rect, r_delay);
+ a->pts = 0.0;
}
- if (got <= 0) {
- MEM_freeN(ib);
+ for (;;) {
+ double pts = -1.0, dur = 0.0;
+ int got = a->codec->next(a->state, dst, &pts, &dur);
+ if (got == 0 && a->index > 0) { /* past the last frame: from the top */
+ a->codec->seek(a->state, 0.0);
+ a->index = 0;
+ a->pts = 0.0;
+ *r_wrapped = true;
+ got = a->codec->next(a->state, dst, &pts, &dur);
+ }
+ if (got <= 0) {
+ a->seek = -1.0;
+ return -1;
+ }
+ if (!a->codec->streamed && dur <= 0.0105) {
+ dur = 0.1; /* GIF/WebP 0 and 10 ms mean "as fast as possible": browsers show 100 ms */
+ }
+ pts = pts >= 0.0 ? pts : a->pts;
+ a->pts = pts + dur;
+ a->index++;
+ a->frames = BK_MAX(a->frames, a->index); /* headers can undercount */
+ /* a seek lands on the frame showing at its time: skip the ones before */
+ if (a->seek >= 0.0 && pts + dur <= a->seek + 1e-6 && !*r_wrapped) {
+ continue;
+ }
+ a->seek = -1.0;
+ *r_pts = pts;
+ *r_dur = dur;
+ return 1;
+ }
+}
+
+/* The next composited frame at full size (a fresh buffer), in-memory codecs. */
+static BKImBuf *next_full(BKImAnim *a, double *r_pts, double *r_dur, bool *r_wrapped)
+{
+ BKImBuf *ib = imb_alloc(a->info.w, a->info.h, 1);
+ if (!ib) {
return NULL;
}
- a->index++;
- a->frames = BK_MAX(a->frames, a->index); /* headers can undercount */
- if (*r_delay <= 10) {
- *r_delay = 100; /* 0 and 10 ms mean "as fast as possible": browsers show 100 ms */
+ ib->ftype = a->ftype;
+ if (codec_next(a, ib, r_pts, r_dur, r_wrapped) < 0) {
+ MEM_freeN(ib);
+ return NULL;
}
return ib;
}
-BKImBuf *imb_anim_next(BKImAnim *a, int *r_delay, bool *r_wrapped)
+BKImBuf *imb_anim_next(BKImAnim *a, BKImBuf *reuse, double *r_pts, double *r_dur, bool *r_wrapped)
{
- BKImBuf *ib = next_full(a, r_delay, r_wrapped);
- return ib ? imb_finish(ib, &a->plan) : NULL;
+ if (!a->codec->streamed) {
+ BK_imbuf_free(reuse); /* composited and fitted into fresh buffers */
+ BKImBuf *ib = next_full(a, r_pts, r_dur, r_wrapped);
+ return ib ? imb_finish(ib, &a->plan) : NULL;
+ }
+ BKImBuf *ib = reuse;
+ if (ib && (ib->x != a->info.w || ib->y != a->info.h || ib->format != a->info.format)) {
+ BK_imbuf_free(ib); /* resized */
+ ib = NULL;
+ }
+ if (!ib) {
+ ib = imb_alloc_nv12(a->info.w, a->info.h, a->info.format);
+ if (!ib) {
+ return NULL;
+ }
+ ib->ftype = BK_IMAGE_VIDEO;
+ ib->stream = a->stream;
+ }
+ if (codec_next(a, ib, r_pts, r_dur, r_wrapped) < 0) {
+ BK_imbuf_free(ib);
+ return NULL;
+ }
+ imb_rekey(ib); /* new pixels: GPU renderers update the stream's texture */
+ /* the whole frame is decoded; the visible part is drawn from it */
+ const double kx = ib->x / (double)a->info.src_w, ky = ib->y / (double)a->info.src_h;
+ ib->src_x = a->info.src_w;
+ ib->src_y = a->info.src_h;
+ ib->crop[0] = (float)(a->plan.src[0] * kx);
+ ib->crop[1] = (float)(a->plan.src[1] * ky);
+ ib->crop[2] = (float)(a->plan.src[2] * kx);
+ ib->crop[3] = (float)(a->plan.src[3] * ky);
+ return ib;
}
BKImBuf *imb_anim_frame(const uint8_t *mem, size_t size, const BKImageFit *target, int frame)
@@ -134,10 +297,10 @@ BKImBuf *imb_anim_frame(const uint8_t *mem, size_t size, const BKImageFit *targe
BKImAnim *a = imb_anim_open(copy, size, target);
BKImBuf *ib = NULL;
for (int i = 0; a && i <= frame; i++) {
- int delay;
+ double pts, dur;
bool wrapped;
MEM_freeN(ib);
- ib = next_full(a, &delay, &wrapped);
+ ib = next_full(a, &pts, &dur, &wrapped);
if (!ib || wrapped) { /* past the end: no such frame */
MEM_freeN(ib);
ib = NULL;
@@ -163,21 +326,22 @@ BKImBuf *imb_anim_all(const uint8_t *mem, size_t size, const BKImageFit *target)
return NULL;
}
/* every frame in one buffer (the synchronous API), at most 128 MiB of them */
+ const int w = a->info.w, h = a->info.h;
const int frames = (int)BK_MIN((size_t)a->frames,
- BK_MAX((size_t)1, ((size_t)1 << 25) / ((size_t)a->w * a->h)));
- BKImBuf *all = imb_alloc(a->w, a->h, frames);
+ BK_MAX((size_t)1, ((size_t)1 << 25) / ((size_t)w * h)));
+ BKImBuf *all = imb_alloc(w, h, frames);
int n = 0;
while (all && n < frames) {
- int delay;
+ double pts, dur;
bool wrapped;
- BKImBuf *ib = next_full(a, &delay, &wrapped);
+ BKImBuf *ib = next_full(a, &pts, &dur, &wrapped);
if (!ib || wrapped) {
MEM_freeN(ib);
break;
}
- memcpy(imb_frame(all, n), ib->rect, sizeof(uint32_t) * (size_t)a->w * a->h);
+ memcpy(imb_frame(all, n), ib->rect, sizeof(uint32_t) * (size_t)w * h);
if (all->delays) {
- all->delays[n] = delay;
+ all->delays[n] = (int)lround(dur * 1000.0);
}
MEM_freeN(ib);
n++;
@@ -188,7 +352,7 @@ BKImBuf *imb_anim_all(const uint8_t *mem, size_t size, const BKImageFit *target)
}
if (all) {
all->frames = n;
- all->loops = a->loops;
+ all->loops = a->info.loops;
all->ftype = a->ftype;
all = imb_finish(all, &a->plan);
}
@@ -213,11 +377,11 @@ BKAnim *BK_anim_open(const void *data, size_t len, int max_w, int max_h)
BKImBuf *BK_anim_next(BKAnim *anim, int *r_delay_ms)
{
- int delay = 0;
+ double pts = 0.0, dur = 0.0;
bool wrapped;
- BKImBuf *ib = anim ? imb_anim_next((BKImAnim *)anim, &delay, &wrapped) : NULL;
+ BKImBuf *ib = anim ? imb_anim_next((BKImAnim *)anim, NULL, &pts, &dur, &wrapped) : NULL;
if (r_delay_ms) {
- *r_delay_ms = delay;
+ *r_delay_ms = (int)lround(dur * 1000.0);
}
return ib;
}
@@ -226,13 +390,13 @@ int BK_anim_info(const BKAnim *anim, int *r_w, int *r_h, int *r_loops)
{
const BKImAnim *a = (const BKImAnim *)anim;
if (r_w) {
- *r_w = a ? a->w : 0;
+ *r_w = a ? a->info.w : 0;
}
if (r_h) {
- *r_h = a ? a->h : 0;
+ *r_h = a ? a->info.h : 0;
}
if (r_loops) {
- *r_loops = a ? a->loops : 0;
+ *r_loops = a ? a->info.loops : 0;
}
return a ? a->frames : 0;
}
diff --git a/kit/intern/image/cache.c b/kit/intern/image/cache.c
index da0fcf1..3ebfed2 100644
--- a/kit/intern/image/cache.c
+++ b/kit/intern/image/cache.c
@@ -10,6 +10,9 @@
* two generations (they are on screen): visible images can exceed the
* budget, the rest can't.
* - Deterministic: BK_image_exit frees everything; MEM counts all of it.
+ * - Private entries (a <video> each, video.c) use the same requests,
+ * clock and visibility tracking, outside the table and the LRU: their
+ * owner frees them.
*
* SPDX-License-Identifier: GPL-2.0-or-later */
@@ -31,23 +34,32 @@ typedef struct Waiter {
unsigned pass; /* the drawing pass that last asked */
} Waiter;
-typedef struct Entry {
- struct Entry *lru_prev, *lru_next; /* most recently used first */
- struct Entry *pend_prev, *pend_next; /* while a request is out */
+typedef struct BKImEntry Entry;
+struct BKImEntry {
+ Entry *lru_prev, *lru_next; /* most recently used first */
+ Entry *pend_prev, *pend_next; /* while a request is out */
BKImageSource *source; /* referenced */
BKImageFit target;
BKImBuf *ib; /* NULL while pending or after a failed decode */
BKImRequest *req; /* the request out (NULL when settled) */
+ BKImRequest *spare; /* a step's request, kept for the next step (no allocation per frame) */
unsigned used; /* generation of the last use */
size_t bytes; /* counted in C.bytes: ib, and play's decoder and next frame */
BKImPlay *play; /* a playing animation (NULL for stills) */
- struct Entry *play_prev, *play_next; /* C.playing */
+ Entry *play_prev, *play_next; /* C.playing */
+ /* private entries (owned: video.c): who hears state changes, and the
+ * last state it heard; orphaned when the owner let go with a request out */
+ bool owned, orphan;
+ unsigned hosted; /* C.gen + 1 when a host loop last showed it without a rect (textures) */
+ void (*notify)(void *owner);
+ void *owner;
+ BKImPlayState heard;
/* where it shows (window rects per context): while pending, what redraws
* on arrival; for animations, what redraws on every frame (a context
* that redraws a rect without asking drops it: off screen, paused) */
int nwaiters;
Waiter waiters[MAX_WAITERS];
-} Entry;
+};
struct BKImageSource {
int refs;
@@ -190,8 +202,10 @@ static void entry_account(Entry *e)
static void entry_free(Entry *e)
{
- BLI_ghash_remove(C.entries, e, NULL, NULL);
- lru_unlink(e);
+ if (!e->owned) {
+ BLI_ghash_remove(C.entries, e, NULL, NULL);
+ lru_unlink(e);
+ }
if (e->play) {
*(e->play_prev ? &e->play_prev->play_next : &C.playing) = e->play_next;
if (e->play_next) {
@@ -202,15 +216,29 @@ static void entry_free(Entry *e)
C.bytes -= e->bytes;
BK_imbuf_free(e->ib);
bk_image_source_release(e->source);
+ MEM_freeN(e->spare);
MEM_freeN(e);
C.count--;
}
-/* `anim`: a step of a playing animation (its next frame), else a first load. */
-static void request(Entry *e, int priority, BKImAnim *anim)
+/* A first load, or (`step`) the next frame of what plays when it has one to decode. */
+static void request(Entry *e, int priority, bool step)
{
- BKImRequest *req = (BKImRequest *)MEM_callocN(sizeof(BKImRequest), "BKImRequest");
+ BKImBuf *reuse = NULL;
+ double seek = -1.0;
+ BKImAnim *anim = step ? imb_play_take_step(e->play, &reuse, &seek) : NULL;
+ if (step && !anim) {
+ return; /* nothing to decode now */
+ }
+ BKImRequest *req = e->spare && step ? e->spare :
+ (BKImRequest *)MEM_mallocN(sizeof(BKImRequest), "BKImRequest");
+ e->spare = req == e->spare ? NULL : e->spare;
+ memset(req, 0, sizeof(*req));
+ req->step = step;
req->anim = anim;
+ req->reuse = reuse;
+ req->seek = seek;
+ req->movie = e->owned && !step;
req->source = e->source;
e->source->refs++; /* the worker reads its path */
req->target = e->target;
@@ -276,7 +304,7 @@ static Entry *lookup(BKImageSource *s, const BKImageFit *target, bool create, in
BLI_ghash_insert(C.entries, e, e);
lru_front(e);
C.count++;
- request(e, priority, NULL);
+ request(e, priority, false);
}
else if (e) {
if (e != C.lru_head) {
@@ -337,18 +365,38 @@ static void arm(const Entry *e)
}
}
+/* A private entry's owner hears every state change (loaded, ended, error). */
+static void heard(Entry *e)
+{
+ const BKImPlayState st = e->play ? imb_play_state(e->play) : BK_PLAY_LOADING;
+ if (e->notify && st != e->heard) {
+ e->heard = st;
+ e->notify(e->owner);
+ }
+}
+
+static void playing_link(Entry *e)
+{
+ e->play_next = C.playing;
+ if (C.playing) {
+ C.playing->play_prev = e;
+ }
+ C.playing = e;
+}
+
/* A step of a playing animation came back. */
static void step_done(Entry *e, BKImRequest *req)
{
if (req->state == BK_IMREQ_CANCELLED) {
- imb_play_step_cancelled(e->play, req->anim);
+ imb_play_step_cancelled(e->play, req->anim, req->reuse);
}
else {
- imb_play_step_done(e->play, req->anim, req->result, req->delay, req->wrapped);
+ imb_play_step_done(e->play, req->anim, req->result, req->pts, req->dur, req->wrapped);
req->result = NULL;
C.stats.decoded++;
}
req->anim = NULL;
+ req->reuse = NULL;
entry_account(e);
arm(e);
}
@@ -360,34 +408,45 @@ int BK_image_poll(void)
while (req) {
BKImRequest *next = req->next;
Entry *e = (Entry *)req->entry;
- if (e && req->state == BK_IMREQ_CANCELLED && e->nwaiters > 0) {
+ Entry *keeper = NULL; /* keeps a step's request for its next step */
+ if (e && !e->orphan && req->state == BK_IMREQ_CANCELLED && e->nwaiters > 0 && !req->step) {
bk_loader_push(req); /* asked for again after the cancel: still wanted */
req = next;
continue;
}
- if (e && e->play) {
+ if (e && e->orphan) { /* its owner let go while this was out */
+ e->req = NULL;
+ pend_unlink(e);
+ entry_free(e);
+ }
+ else if (e && req->step) {
e->req = NULL;
pend_unlink(e);
step_done(e, req);
+ keeper = e->spare ? NULL : e;
}
else if (e) {
e->req = NULL;
pend_unlink(e);
if (req->state == BK_IMREQ_CANCELLED) {
- entry_free(e);
+ if (!e->owned) {
+ entry_free(e);
+ }
C.stats.cancelled++;
}
else {
e->ib = req->result;
req->result = NULL;
if (e->ib && req->anim) { /* it plays: keep the decoder, show where it is */
- e->play = imb_play_new(req->anim, req->delay);
- req->anim = NULL;
- e->play_next = C.playing;
- if (C.playing) {
- C.playing->play_prev = e;
+ if (!e->play) {
+ e->play = imb_play_new(false, -1);
+ playing_link(e);
}
- C.playing = e;
+ imb_play_start(e->play, req->anim, req->pts, req->dur);
+ req->anim = NULL;
+ }
+ else if (e->play) {
+ imb_play_fail(e->play); /* a video that didn't open */
}
entry_account(e);
if (e->ib) {
@@ -404,13 +463,20 @@ int BK_image_poll(void)
if (!e->play) {
e->nwaiters = 0; /* a still: nothing more to redraw */
}
+ heard(e);
n++;
}
}
BK_imbuf_free(req->result);
+ BK_imbuf_free(req->reuse);
imb_anim_free(req->anim); /* a first load whose entry is gone */
bk_image_source_release(req->source);
- MEM_freeN(req);
+ if (keeper) {
+ keeper->spare = req;
+ }
+ else {
+ MEM_freeN(req);
+ }
req = next;
}
return n;
@@ -423,23 +489,21 @@ void bk_image_tick(void)
}
const double now = bk_time() + 0.002; /* timers fire a little early */
for (Entry *e = C.playing; e; e = e->play_next) {
- if (e->nwaiters == 0) {
+ if (e->nwaiters == 0 && !(e->hosted && C.gen + 1 - e->hosted <= 1)) {
continue; /* nobody shows it: paused, no decoding */
}
BKImBuf *frame = imb_play_advance(e->play, now);
if (frame) {
- BK_imbuf_free(e->ib);
+ imb_play_recycle(e->play, e->ib); /* a movie decodes its next frame into it */
e->ib = frame;
entry_account(e);
damage(e); /* only its rectangles redraw */
}
if (!e->req) {
- BKImAnim *a = imb_play_take_step(e->play);
- if (a) {
- request(e, BK_IMAGE_PRIORITY_VISIBLE, a); /* the one after, while this shows */
- }
+ request(e, BK_IMAGE_PRIORITY_VISIBLE, true); /* the one after, while this shows */
}
arm(e);
+ heard(e);
}
}
@@ -562,6 +626,91 @@ void bk_image_forget_context(BKContext *ctx)
/** \} */
+/* -------------------------------------------------------------------- */
+/** \name Private entries (video.c)
+ * \{ */
+
+BKImEntry *bk_image_entry_new(BKImageSource *s, BKImPlay *play, void (*notify)(void *owner),
+ void *owner)
+{
+ Entry *e = (Entry *)MEM_callocN(sizeof(Entry), "image private entry");
+ e->owned = true;
+ e->source = s;
+ s->refs++;
+ e->play = play;
+ e->notify = notify;
+ e->owner = owner;
+ playing_link(e);
+ C.count++;
+ entry_account(e);
+ return e;
+}
+
+void bk_image_entry_free(BKImEntry *e)
+{
+ if (!e) {
+ return;
+ }
+ if (e->req) {
+ e->orphan = true; /* freed when the request comes back (BK_image_poll) */
+ e->notify = NULL;
+ e->nwaiters = 0;
+ bk_loader_cancel(e->req);
+ return;
+ }
+ entry_free(e);
+}
+
+/* Decoding at about the size drawn: a box much bigger, or much smaller, than
+ * what the decoder makes asks for another size (resizing within that is
+ * scaled when drawn, so a window resize doesn't reopen the decoder). */
+static bool retarget_wanted(const BKImageFit *had, const BKImageFit *want)
+{
+ return had->fit != want->fit || want->box_w * 4 > had->box_w * 5 ||
+ want->box_h * 4 > had->box_h * 5 || want->box_w * 2 < had->box_w ||
+ want->box_h * 2 < had->box_h;
+}
+
+const BKImBuf *bk_image_entry_get(BKImEntry *e, const BKImageFit *target, const int rect[4])
+{
+ const bool loaded = imb_play_state(e->play) != BK_PLAY_LOADING;
+ if ((!loaded && !e->req && imb_play_state(e->play) != BK_PLAY_ERROR) ||
+ retarget_wanted(&e->target, target))
+ {
+ e->target = *target; /* the first load, or the next step, decodes at this size */
+ }
+ if (!loaded && !e->req && imb_play_state(e->play) != BK_PLAY_ERROR) {
+ request(e, BK_IMAGE_PRIORITY_VISIBLE, false);
+ }
+ if (rect) {
+ add_waiter(e, rect);
+ }
+ else {
+ e->hosted = C.gen + 1;
+ }
+ if (e->ib) {
+ imb_play_drawn(e->play, bk_time());
+ }
+ return e->ib;
+}
+
+BKImPlay *bk_image_entry_play(BKImEntry *e)
+{
+ return e->play;
+}
+
+void bk_image_entry_changed(BKImEntry *e)
+{
+ if (!e->req) {
+ request(e, BK_IMAGE_PRIORITY_VISIBLE, true); /* a seek decodes now */
+ }
+ damage(e);
+ arm(e);
+ heard(e);
+}
+
+/** \} */
+
/* -------------------------------------------------------------------- */
/** \name Settings, statistics, exit (BUI_kit.h)
* \{ */
diff --git a/kit/intern/image/draw.c b/kit/intern/image/draw.c
index 724ad53..bad4338 100644
--- a/kit/intern/image/draw.c
+++ b/kit/intern/image/draw.c
@@ -1,7 +1,8 @@
/* Images on nodes and canvas items: what a BKImageRef shows, fitted into a
* box (CSS object-fit, the same bk_image_fit the decoder used), drawn with
* BUI_raster_image. Sources go through the cache: drawn when decoded,
- * requested at the box size otherwise (the box redraws on arrival).
+ * requested at the box size otherwise (the box redraws on arrival). Videos
+ * (video.c) show their current frame the same way.
*
* SPDX-License-Identifier: GPL-2.0-or-later */
@@ -71,6 +72,18 @@ void bk_image_ref_set_imbuf(BKImageRef *ref, const BKImBuf *ib)
}
}
+void bk_image_ref_set_video(BKImageRef *ref, BKVideo *v)
+{
+ if (ref->kind == BK_IMREF_VIDEO && ref->u.video == v) {
+ return;
+ }
+ bk_image_ref_clear(ref);
+ if (v) {
+ ref->kind = BK_IMREF_VIDEO;
+ ref->u.video = v;
+ }
+}
+
void bk_image_ref_set_source(BKImageRef *ref, const char *src)
{
if (ref->kind == BK_IMREF_SOURCE && src &&
@@ -102,6 +115,9 @@ bool bk_image_ref_size(const BKImageRef *ref, int *r_w, int *r_h)
*r_h = ib->y;
return true;
}
+ if (ref->kind == BK_IMREF_VIDEO) {
+ return bk_video_size(ref->u.video, r_w, r_h);
+ }
if (ref->kind == BK_IMREF_SOURCE && bk_image_source_info(ref->u.source, &info)) {
*r_w = info.width;
*r_h = info.height;
@@ -116,10 +132,11 @@ bool bk_image_ref_resolve(const BKImageRef *ref, int fit, int frame, const float
{
const BKImageFit target = {fit, BK_MAX(1, (int)lroundf(box[2])), BK_MAX(1, (int)lroundf(box[3])),
0.5f, 0.5f, BK_MAX(0, frame)};
- if (ref->kind == BK_IMREF_SOURCE) {
+ if (ref->kind == BK_IMREF_SOURCE || ref->kind == BK_IMREF_VIDEO) {
const int rect[4] = {(int)floorf(box[0]), (int)floorf(box[1]), (int)ceilf(box[0] + box[2]),
(int)ceilf(box[1] + box[3])};
- r->ib = bk_image_cache_get(ref->u.source, &target, rect);
+ r->ib = ref->kind == BK_IMREF_VIDEO ? bk_video_frame(ref->u.video, &target, rect) :
+ bk_image_cache_get(ref->u.source, &target, rect);
if (!r->ib) {
return false;
}
@@ -142,7 +159,7 @@ bool bk_image_ref_resolve(const BKImageRef *ref, int fit, int frame, const float
void bk_image_draw_resolved(BuiRaster *r, const BKImageDraw *d, float alpha)
{
- const BuiPixmap pm = {d->ib->rect, d->ib->x, d->ib->y, d->ib->key};
+ const BuiPixmap pm = imb_pixmap(d->ib);
BUI_raster_image(r, d->dst[0], d->dst[1], d->dst[2], d->dst[3], &pm, d->src, alpha, 0);
}
@@ -156,7 +173,7 @@ void bk_image_ref_draw(const BKImageRef *ref, int fit, int frame, const float bo
void bk_imbuf_draw(const BKImBuf *ib, float x, float y, float w, float h, float alpha)
{
- const BuiPixmap pm = {ib->rect, ib->x, ib->y, ib->key};
+ const BuiPixmap pm = imb_pixmap(ib);
BUI_raster_image(bk_draw.r, x, y, w, h, &pm, NULL, alpha, 0);
}
@@ -246,20 +263,29 @@ const BKClass bk_cls_image = {.name = "image", .measure = image_measure, .draw =
* holds straight RGBA (Dear ImGui's convention). */
static int texture_put(const BKImBuf *ib, int frame, int tex)
{
- const uint32_t *px = BK_imbuf_pixels(ib, frame);
int tw, th;
- if (!px || (tex && (!BUI_texture_size(tex, &tw, &th) || tw != ib->x || th != ib->y))) {
+ if (!ib || frame < 0 || frame >= ib->frames ||
+ (tex && (!BUI_texture_size(tex, &tw, &th) || tw != ib->x || th != ib->y)))
+ {
return 0;
}
- unsigned char *rgba = (unsigned char *)MEM_mallocN((size_t)ib->x * ib->y * 4, "texture rgba");
- if (!rgba) {
+ /* bands of rows through the stack: no allocation per frame (video textures) */
+ uint8_t band[64 * 1024];
+ const int step = (int)sizeof(band) / (ib->x * 4);
+ if (step < 1) {
return 0;
}
- imb_unpremultiply(rgba, px, ib->x * ib->y);
- const int ok = tex ? BUI_texture_update(tex, 0, 0, ib->x, ib->y, rgba, 0) :
- BUI_texture_create(ib->x, ib->y, rgba, 0);
- MEM_freeN(rgba);
- return ok;
+ const int t = tex ? tex : BUI_texture_create(ib->x, ib->y, NULL, 0);
+ for (int y = 0; t && y < ib->y; y += step) {
+ const int n = BK_MIN(step, ib->y - y);
+ if (!imb_rgba_rows(ib, frame, y, n, band) || !BUI_texture_update(t, 0, y, ib->x, n, band, 0)) {
+ if (!tex) {
+ BUI_texture_free(t);
+ }
+ return 0;
+ }
+ }
+ return t;
}
int BK_imbuf_texture(const BKImBuf *ib, int frame)
diff --git a/kit/intern/image/format_webp.c b/kit/intern/image/format_webp.c
index be1ceca..41cfcc6 100644
--- a/kit/intern/image/format_webp.c
+++ b/kit/intern/image/format_webp.c
@@ -95,8 +95,12 @@ static void webp_close(void *state)
MEM_freeN(a);
}
-static void *webp_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *r_loops)
+static void *webp_open(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,
+ BKImAnimInfo *r_info)
{
+ (void)path;
+ (void)want_w;
+ (void)want_h;
WebPAnimDecoderOptions opt;
if (!WebPAnimDecoderOptionsInit(&opt)) {
return NULL;
@@ -114,14 +118,16 @@ static void *webp_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int
webp_close(a);
return NULL;
}
- *r_w = a->w = (int)info.canvas_width;
- *r_h = a->h = (int)info.canvas_height;
- *r_loops = (int)info.loop_count;
+ r_info->w = r_info->src_w = a->w = (int)info.canvas_width;
+ r_info->h = r_info->src_h = a->h = (int)info.canvas_height;
+ r_info->format = BUI_PIXMAP_ARGB;
+ r_info->loops = (int)info.loop_count;
return a;
}
-static int webp_next(void *state, uint32_t *dst, int *r_delay)
+static int webp_next(void *state, BKImBuf *dst, double *r_pts, double *r_dur)
{
+ (void)r_pts;
WebpAnim *a = (WebpAnim *)state;
uint8_t *buf;
int ts;
@@ -131,17 +137,19 @@ static int webp_next(void *state, uint32_t *dst, int *r_delay)
if (!WebPAnimDecoderGetNext(a->dec, &buf, &ts)) {
return -1;
}
- memcpy(dst, buf, sizeof(uint32_t) * (size_t)a->w * a->h);
- *r_delay = ts - a->prev_ts; /* timestamps are frame ends */
+ memcpy(dst->rect, buf, sizeof(uint32_t) * (size_t)a->w * a->h);
+ *r_dur = (ts - a->prev_ts) / 1000.0; /* timestamps (ms) are frame ends */
a->prev_ts = ts;
return 1;
}
-static void webp_rewind(void *state)
+static bool webp_seek(void *state, double t)
{
+ (void)t; /* from the top: anim.c skips to t */
WebpAnim *a = (WebpAnim *)state;
WebPAnimDecoderReset(a->dec);
a->prev_ts = 0;
+ return true;
}
static size_t webp_bytes(const void *state)
@@ -150,6 +158,6 @@ static size_t webp_bytes(const void *state)
return sizeof(*a) + 2 * sizeof(uint32_t) * (size_t)a->w * a->h; /* its canvas and previous */
}
-const BKImAnimCodec imb_webp_codec = {webp_open, webp_next, webp_rewind, webp_bytes, webp_close};
+const BKImAnimCodec imb_webp_codec = {"webp", false, imb_webp_animated, webp_open, webp_next, webp_seek, webp_bytes, webp_close};
/** \} */
diff --git a/kit/intern/image/format_wuffs.c b/kit/intern/image/format_wuffs.c
index 19b9007..7acb33a 100644
--- a/kit/intern/image/format_wuffs.c
+++ b/kit/intern/image/format_wuffs.c
@@ -272,8 +272,12 @@ static void gif_close(void *state)
MEM_freeN(g);
}
-static void *gif_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *r_loops)
+static void *gif_open(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,
+ BKImAnimInfo *r_info)
{
+ (void)path;
+ (void)want_w;
+ (void)want_h;
GifAnim *g = (GifAnim *)MEM_callocN(sizeof(GifAnim), "GifAnim");
g->mem = mem;
g->size = size;
@@ -292,8 +296,9 @@ static void *gif_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *
gif_close(g);
return NULL;
}
- *r_w = g->w;
- *r_h = g->h;
+ r_info->w = r_info->src_w = g->w;
+ r_info->h = r_info->src_h = g->h;
+ r_info->format = BUI_PIXMAP_ARGB;
/* plays (0: forever): known once the frames were seen (no NETSCAPE block:
* one play), so ask a decoder that skipped through them */
wuffs_base__io_buffer src;
@@ -301,19 +306,21 @@ static void *gif_open(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *
wuffs_base__image_decoder *count = open_image(mem, size, &src, &ic);
if (count) {
count_frames(count, &src);
- *r_loops = (int)wuffs_base__image_decoder__num_animation_loops(count);
+ r_info->loops = (int)wuffs_base__image_decoder__num_animation_loops(count);
MEM_freeN(count);
}
return g;
}
-static void gif_rewind(void *state)
+static bool gif_seek(void *state, double t)
{
- gif_start((GifAnim *)state);
+ (void)t; /* from the top: anim.c skips to t */
+ return gif_start((GifAnim *)state);
}
-static int gif_next(void *state, uint32_t *dst, int *r_delay)
+static int gif_next(void *state, BKImBuf *dst, double *r_pts, double *r_dur)
{
+ (void)r_pts;
GifAnim *g = (GifAnim *)state;
if (!g->dec) {
return -1;
@@ -343,10 +350,10 @@ static int gif_next(void *state, uint32_t *dst, int *r_delay)
if (frame.repr && g->index == 0) {
return -1; /* a bad first frame: no image (later bad frames end the animation) */
}
- *r_delay = (int)(wuffs_base__frame_config__duration(&fc) / WUFFS_BASE__FLICKS_PER_MILLISECOND);
+ *r_dur = (double)wuffs_base__frame_config__duration(&fc) / (double)WUFFS_BASE__FLICKS_PER_SECOND;
g->prev = fc;
g->index++;
- memcpy(dst, g->canvas, sizeof(uint32_t) * (size_t)g->w * g->h);
+ memcpy(dst->rect, g->canvas, sizeof(uint32_t) * (size_t)g->w * g->h);
return 1;
}
@@ -356,6 +363,6 @@ static size_t gif_bytes(const void *state)
return sizeof(*g) + (size_t)g->wl + 2 * sizeof(uint32_t) * (size_t)g->w * g->h;
}
-const BKImAnimCodec imb_gif_codec = {gif_open, gif_next, gif_rewind, gif_bytes, gif_close};
+const BKImAnimCodec imb_gif_codec = {"gif", false, imb_is_a_gif, gif_open, gif_next, gif_seek, gif_bytes, gif_close};
/** \} */
diff --git a/kit/intern/image/image.h b/kit/intern/image/image.h
index 1d56693..992cf4a 100644
--- a/kit/intern/image/image.h
+++ b/kit/intern/image/image.h
@@ -40,7 +40,9 @@
struct BKImBuf {
int x, y; /* size in pixels (ImBuf::x, ::y) */
int frames; /* >= 1 */
- int ftype; /* BK_IMAGE_* it was decoded from */
+ int ftype; /* BK_IMAGE_* it was decoded from (BK_IMAGE_VIDEO: a movie frame) */
+ int format; /* BUI_PIXMAP_*: ARGB, or NV12 (one frame; `rect` holds its planes) */
+ uint64_t stream; /* frames of one playing movie share it (one GPU texture); 0 else */
int src_x, src_y; /* intrinsic size of the source (after EXIF orientation) */
float crop[4]; /* the part to draw (x, y, w, h in these pixels): the visible
* region of a fitted decode, sub-pixel exact; else all */
@@ -56,6 +58,16 @@ struct BKImBuf {
/* One block: header, delays, pixels (uninitialized). NULL when the size is
* out of range or allocation fails. */
BKImBuf *imb_alloc(int x, int y, int frames);
+/* One NV12 frame (`format` BUI_PIXMAP_NV12_*; x and y even): luma, then chroma. */
+BKImBuf *imb_alloc_nv12(int x, int y, int format);
+/* A fresh unique key (BKImBuf keys, movie streams). */
+uint64_t imb_new_key(void);
+/* What the GPU renderers draw for `ib`. */
+static inline BuiPixmap imb_pixmap(const BKImBuf *ib)
+{
+ const BuiPixmap pm = {ib->rect, ib->x, ib->y, ib->key, ib->format, ib->stream};
+ return pm;
+}
static inline uint32_t *imb_frame(const BKImBuf *ib, int frame)
{
return ib->rect + (size_t)frame * ib->x * ib->y;
@@ -72,6 +84,8 @@ void imb_rekey(BKImBuf *ib);
/* Straight RGBA (8 bits, any stride) <-> premultiplied 0xAARRGGBB rows. */
void imb_premultiply_rgba(uint32_t *dst, const uint8_t *rgba, int n);
void imb_unpremultiply(uint8_t *rgba, const uint32_t *src, int n);
+/* Rows y0 .. y0 + rows of a frame as straight RGBA (any format). */
+bool imb_rgba_rows(const BKImBuf *ib, int frame, int y0, int rows, uint8_t *dst);
/* Where an image of intrinsic size iw x ih shows in a bw x bh box (CSS
* object-fit / object-position): `dst` is the visible rect inside the box
@@ -122,31 +136,71 @@ BKImBuf *imb_load(const uint8_t *mem, size_t size, const BKImageFit *target);
/* Reads a whole file (MEM_mallocN'd), or its first `max` bytes (0: all). */
uint8_t *imb_read_file(const char *path, size_t max, size_t *r_size);
-/* -- anim.c: animations, one frame at a time ---------------------------------------- */
+/* -- anim.c: animated media, one frame at a time --------------------------------------
+ * GIF, animated WebP and video share one codec interface (Blender's anim /
+ * MovieReader: open, next frame, seek, close) and one player (play.c). */
+
+typedef struct BKImAnimInfo {
+ int w, h; /* the frames' size: the canvas (in-memory codecs) or what a
+ * streamed codec decodes to (about the size asked, even) */
+ int src_w, src_h; /* intrinsic (display) size */
+ int format; /* BUI_PIXMAP_* of the frames */
+ int loops; /* plays the file asks for, 0 = forever */
+ double duration; /* seconds, 0 when unknown */
+} BKImAnimInfo;
-/* What a format provides for animations: frames composited (disposal,
- * blending) into a full-size premultiplied canvas copy. */
typedef struct BKImAnimCodec {
- void *(*open)(const uint8_t *mem, size_t size, int *r_w, int *r_h, int *r_loops);
- /* 1 with the next frame in `dst` (w * h), 0 past the last, -1 on errors */
- int (*next)(void *state, uint32_t *dst, int *r_delay_ms);
- void (*rewind)(void *state);
- size_t (*bytes)(const void *state);
+ const char *name;
+ /* Streamed codecs (video) read the file themselves, by path, as it plays:
+ * nothing is loaded whole. Others get the whole file in memory. */
+ bool streamed;
+ /* The file starts like one this codec reads (a few dozen bytes). */
+ bool (*is_a)(const uint8_t *head, size_t size);
+ /* `path` (streamed) or `mem` (kept by the caller while open). want_w x
+ * want_h is the size frames are drawn at: a hint (video scales to it on
+ * the decoder, in hardware where there is one). */
+ void *(*open)(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,
+ BKImAnimInfo *r_info);
+ /* 1 with the next frame in `dst` (the info's size and format), its time
+ * and duration in seconds (pts < 0: unknown, frames follow each other);
+ * 0 past the last; -1 on errors. */
+ int (*next)(void *state, BKImBuf *dst, double *r_pts, double *r_dur);
+ /* The next frame is the one showing at `t`, or one before it (a keyframe;
+ * in-memory codecs restart): anim.c skips forward to `t`. */
+ bool (*seek)(void *state, double t);
+ size_t (*bytes)(const void *state); /* decoder state (estimate for OS decoders) */
void (*close)(void *state);
} BKImAnimCodec;
extern const BKImAnimCodec imb_gif_codec, imb_webp_codec;
-/* The codec for an animation (GIF always, animated WebP). */
+/* Video backends (movie_*.c), each compiled when the build has it. */
+extern const BKImAnimCodec imb_movie_mf, imb_movie_avf, imb_movie_gst, imb_movie_dav1d,
+ imb_movie_ffmpeg;
+/* Those compiled in (movie.c: the build's choice), tried in order; NULL-ended. */
+extern const BKImAnimCodec *const imb_movie_codecs[];
+/* The first bytes look like a movie container (MP4/MOV, WebM/MKV, AVI, ASF,
+ * Ogg, MPEG-PS/TS, FLV). */
+bool imb_is_a_movie(const uint8_t *head, size_t size);
+/* The in-memory codec for an animation (GIF always, animated WebP). */
const BKImAnimCodec *imb_anim_codec(const uint8_t *mem, size_t size);
typedef struct BKImAnim BKImAnim;
/* Takes `file` (MEM-allocated; freed with the animation, or now on failure). */
BKImAnim *imb_anim_open(uint8_t *file, size_t size, const BKImageFit *target);
+/* A movie (streamed codecs; NULL when none reads the file). */
+BKImAnim *imb_movie_open(const char *path, const BKImageFit *target);
void imb_anim_free(BKImAnim *a);
-int imb_anim_loops(const BKImAnim *a);
+const BKImAnimInfo *imb_anim_info(const BKImAnim *a);
int imb_anim_frames(const BKImAnim *a); /* from the header (at least what was decoded) */
size_t imb_anim_bytes(const BKImAnim *a); /* its decoder state and file */
-/* The next frame at the target size, wrapping (`r_wrapped`) after the last. */
-BKImBuf *imb_anim_next(BKImAnim *a, int *r_delay_ms, bool *r_wrapped);
+/* Frames to `target` from now on (a movie reopens its decoder at the new
+ * size, where it was). False when it can't (the old size stays). */
+bool imb_anim_retarget(BKImAnim *a, const BKImageFit *target);
+/* The next frame at the target size, wrapping (`r_wrapped`) after the last.
+ * `reuse` (taken; NULL for none): a frame no longer shown, which a movie
+ * decodes into when it fits (no allocation per frame). */
+BKImBuf *imb_anim_next(BKImAnim *a, BKImBuf *reuse, double *r_pts, double *r_dur, bool *r_wrapped);
+/* The next imb_anim_next returns the frame showing at `t` seconds. */
+void imb_anim_seek(BKImAnim *a, double t);
/* Frame `frame` alone, or every frame (synchronous API; at most 128 MiB). */
BKImBuf *imb_anim_frame(const uint8_t *mem, size_t size, const BKImageFit *target, int frame);
BKImBuf *imb_anim_all(const uint8_t *mem, size_t size, const BKImageFit *target);
@@ -218,20 +272,60 @@ double bk_image_next_due(BKContext *ctx);
const BKImBuf *bk_image_cache_get(BKImageSource *s, const BKImageFit *target, const int rect[4]);
void bk_image_cache_prefetch(BKImageSource *s, const BKImageFit *target);
-/* play.c: the frame clock of a playing animation (owned by its cache entry). */
-typedef struct BKImPlay BKImPlay;
-BKImPlay *imb_play_new(BKImAnim *anim, int delay); /* takes `anim`; shows its first frame */
+/* A private entry: one player's (a <video>), outside the shared table and
+ * the LRU (never evicted), freed by its owner (video.c). It loads at its
+ * first draw, with `play` (taken) as its clock; `notify` hears every state
+ * change of the clock (loaded, ended, error) on the UI thread. */
+typedef struct BKImEntry BKImEntry;
+typedef struct BKImPlay BKImPlay; /* play.c */
+BKImEntry *bk_image_entry_new(BKImageSource *s, BKImPlay *play, void (*notify)(void *owner),
+ void *owner);
+void bk_image_entry_free(BKImEntry *e);
+/* As bk_image_cache_get; the entry follows the size it is drawn at. */
+const BKImBuf *bk_image_entry_get(BKImEntry *e, const BKImageFit *target, const int rect[4]);
+BKImPlay *bk_image_entry_play(BKImEntry *e);
+/* A control changed the clock: step now, redraw where it shows. */
+void bk_image_entry_changed(BKImEntry *e);
+
+/* play.c: the one clock of everything that plays (GIF, WebP, video), owned
+ * by its cache entry. Frames advance only after the shown one was drawn. */
+typedef enum BKImPlayState {
+ BK_PLAY_LOADING = 0, /* no frame yet */
+ BK_PLAY_PLAYING,
+ BK_PLAY_PAUSED,
+ BK_PLAY_ENDED, /* the last play's last frame shows */
+ BK_PLAY_ERROR,
+} BKImPlayState;
+/* `loops` overrides the file's play count (-1: the file's; 0: forever). */
+BKImPlay *imb_play_new(bool paused, int loops);
void imb_play_free(BKImPlay *p);
size_t imb_play_bytes(const BKImPlay *p);
-/* The decoder for the next step request (NULL: nothing to decode now). */
-BKImAnim *imb_play_take_step(BKImPlay *p);
+/* The first frame arrived with its decoder (taken). */
+void imb_play_start(BKImPlay *p, BKImAnim *anim, double pts, double dur);
+void imb_play_fail(BKImPlay *p); /* the file didn't open */
+/* A step request to make: the decoder (NULL: nothing to decode now), the
+ * buffer to decode into (or NULL) and where to seek first (< 0: no seek).
+ * The step owns them until it comes back (step_done / step_cancelled). */
+BKImAnim *imb_play_take_step(BKImPlay *p, BKImBuf **r_reuse, double *r_seek);
/* The step came back: the decoder, and the next frame (NULL: an error). */
-void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, int delay, bool wrapped);
-void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim); /* the decoder back, no frame */
+void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, double pts, double dur,
+ bool wrapped);
+void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim, BKImBuf *reuse); /* nothing decoded */
void imb_play_drawn(BKImPlay *p, double now); /* the shown frame reached the screen */
double imb_play_due(const BKImPlay *p); /* when to advance (0: not waiting on time) */
-/* The frame to show from `now` (the caller frees the old one), or NULL. */
+/* The frame to show from `now`, or NULL; the one it replaces goes back
+ * through imb_play_recycle. */
BKImBuf *imb_play_advance(BKImPlay *p, double now);
+void imb_play_recycle(BKImPlay *p, BKImBuf *old); /* kept for a movie's next decode, else freed */
+/* Controls (the <video> element's). */
+void imb_play_pause(BKImPlay *p, bool paused);
+void imb_play_seek(BKImPlay *p, double t);
+void imb_play_set_rate(BKImPlay *p, double rate); /* > 0 */
+void imb_play_set_loops(BKImPlay *p, int loops); /* 0: forever, -1: the file's */
+BKImPlayState imb_play_state(const BKImPlay *p);
+double imb_play_time(const BKImPlay *p); /* seconds, of the frame shown */
+double imb_play_duration(const BKImPlay *p); /* seconds, 0 when unknown */
+bool imb_play_size(const BKImPlay *p, int *r_w, int *r_h); /* intrinsic, once loaded */
/* draw.c: what a node or canvas item shows. One of an owned copy (set from
* RGBA), a borrowed BKImBuf (the host keeps it alive) or a source decoded
@@ -241,6 +335,7 @@ typedef enum BKImageRefKind {
BK_IMREF_OWNED,
BK_IMREF_BORROWED,
BK_IMREF_SOURCE,
+ BK_IMREF_VIDEO, /* a BKVideo (borrowed: its owner frees it) */
} BKImageRefKind;
typedef struct BKImageRef {
BKImageRefKind kind;
@@ -248,6 +343,7 @@ typedef struct BKImageRef {
BKImBuf *owned;
const BKImBuf *borrowed;
BKImageSource *source;
+ BKVideo *video;
} u;
} BKImageRef;
void bk_image_ref_clear(BKImageRef *ref);
@@ -255,6 +351,12 @@ void bk_image_ref_clear(BKImageRef *ref);
void bk_image_ref_set_rgba(BKImageRef *ref, const unsigned char *rgba, int w, int h);
void bk_image_ref_set_imbuf(BKImageRef *ref, const BKImBuf *ib);
void bk_image_ref_set_source(BKImageRef *ref, const char *src);
+void bk_image_ref_set_video(BKImageRef *ref, BKVideo *v);
+
+/* video.c: a BKVideo's frame for drawing at `target` (NULL until the first
+ * frame; then its poster, if any, is what shows), and its intrinsic size. */
+const BKImBuf *bk_video_frame(BKVideo *v, const BKImageFit *target, const int rect[4]);
+bool bk_video_size(const BKVideo *v, int *r_w, int *r_h);
/* Intrinsic size (a source reads its header once). */
bool bk_image_ref_size(const BKImageRef *ref, int *r_w, int *r_h);
@@ -284,9 +386,13 @@ struct BKImRequest {
BKImBuf *result; /* NULL when decoding failed */
void *entry; /* the cache entry waiting for it (cache.c) */
BKImAnim *anim; /* a step: the animation to decode the next frame of; a
- * first load: set when the file plays (frames > 1) */
- int delay; /* the result's delay (ms) and whether it starts a new play */
- bool wrapped;
+ * first load: set when the file plays (frames > 1, movies) */
+ BKImBuf *reuse; /* a step: the frame to decode into (taken) */
+ double seek; /* a step: seek there first (seconds; < 0: no seek) */
+ bool step; /* the next frame of something playing (else a first load) */
+ bool movie; /* a first load of a movie (streamed codecs, imb_movie_open) */
+ double pts, dur; /* the result's time and duration (seconds), and whether */
+ bool wrapped; /* it starts a new play */
};
enum { BK_IMREQ_QUEUED, BK_IMREQ_RUNNING, BK_IMREQ_DONE, BK_IMREQ_CANCELLED };
/* Queue `req` (ownership moves to the loader until it comes back). */
diff --git a/kit/intern/image/imbuf.c b/kit/intern/image/imbuf.c
index 786c05f..0e48a8b 100644
--- a/kit/intern/image/imbuf.c
+++ b/kit/intern/image/imbuf.c
@@ -19,7 +19,7 @@
/** \name Allocation
* \{ */
-static uint64_t next_key(void)
+uint64_t imb_new_key(void)
{
static volatile uint64_t key = 0;
#ifdef _MSC_VER
@@ -48,7 +48,7 @@ BKImBuf *imb_alloc(int x, int y, int frames)
ib->x = ib->src_x = x;
ib->y = ib->src_y = y;
ib->frames = frames;
- ib->key = next_key();
+ ib->key = imb_new_key();
ib->size = size;
ib->delays = frames > 1 ? (int *)(ib + 1) : NULL;
ib->rect = (uint32_t *)((char *)ib + head);
@@ -57,9 +57,33 @@ BKImBuf *imb_alloc(int x, int y, int frames)
return ib;
}
+BKImBuf *imb_alloc_nv12(int x, int y, int format)
+{
+ if (x <= 0 || y <= 0 || (x | y) & 1 || x > BK_IMB_MAX_SIDE || y > BK_IMB_MAX_SIDE) {
+ return NULL;
+ }
+ const size_t head = (sizeof(BKImBuf) + 15) & ~(size_t)15;
+ const size_t size = head + ((size_t)x * y * 3 / 2 + 3) / 4 * 4;
+ BKImBuf *ib = (BKImBuf *)MEM_mallocN(size, "BKImBuf NV12");
+ if (!ib) {
+ return NULL;
+ }
+ memset(ib, 0, sizeof(*ib));
+ ib->x = ib->src_x = x;
+ ib->y = ib->src_y = y;
+ ib->frames = 1;
+ ib->format = format;
+ ib->key = imb_new_key();
+ ib->size = size;
+ ib->rect = (uint32_t *)((char *)ib + head);
+ ib->crop[2] = (float)x;
+ ib->crop[3] = (float)y;
+ return ib;
+}
+
void imb_rekey(BKImBuf *ib)
{
- ib->key = next_key();
+ ib->key = imb_new_key();
}
/* A new buffer of w x h carrying `ib`'s metadata (frames, delays, source). */
@@ -80,8 +104,8 @@ static BKImBuf *imb_alloc_like(const BKImBuf *ib, int w, int h)
void BK_imbuf_free(BKImBuf *ib)
{
- if (ib && (ib->x > BUI_IMAGE_ATLAS_MAX || ib->y > BUI_IMAGE_ATLAS_MAX)) {
- BUI_image_forget(ib->key); /* its own GPU texture goes with it */
+ if (ib && !ib->stream && (ib->x > BUI_IMAGE_ATLAS_MAX || ib->y > BUI_IMAGE_ATLAS_MAX)) {
+ BUI_image_forget(ib->key); /* its own GPU texture goes with it (a stream's, with its movie) */
}
MEM_freeN(ib); /* one block (MEM_freeN(NULL) is a no-op) */
}
@@ -401,7 +425,9 @@ int BK_imbuf_format(const BKImBuf *ib)
const uint32_t *BK_imbuf_pixels(const BKImBuf *ib, int frame)
{
- return ib && frame >= 0 && frame < ib->frames ? imb_frame(ib, frame) : NULL;
+ return ib && ib->format == BUI_PIXMAP_ARGB && frame >= 0 && frame < ib->frames ?
+ imb_frame(ib, frame) :
+ NULL;
}
int BK_imbuf_delay(const BKImBuf *ib, int frame)
@@ -414,14 +440,31 @@ int BK_imbuf_loops(const BKImBuf *ib)
return ib ? ib->loops : 0;
}
-int BK_imbuf_rgba(const BKImBuf *ib, int frame, unsigned char *dst)
+bool imb_rgba_rows(const BKImBuf *ib, int frame, int y0, int rows, uint8_t *dst)
{
- const uint32_t *px = BK_imbuf_pixels(ib, frame);
- if (!px || !dst) {
- return 0;
+ if (ib->format == BUI_PIXMAP_ARGB) {
+ imb_unpremultiply(dst, imb_frame(ib, frame) + (size_t)y0 * ib->x, ib->x * rows);
+ return true;
+ }
+ /* a video frame: converted a band of rows at a time (no allocation) */
+ const BuiPixmap pm = imb_pixmap(ib);
+ uint32_t band[8192];
+ const int cap = (int)(sizeof(band) / sizeof(band[0]));
+ if (ib->x > cap) {
+ return false; /* wider than a band: not a size movies decode to */
+ }
+ const int step = cap / ib->x;
+ for (int y = y0; y < y0 + rows; y += step) {
+ const int n = BK_MIN(step, y0 + rows - y);
+ BUI_pixmap_read(&pm, y, n, band);
+ imb_unpremultiply(dst + (size_t)(y - y0) * ib->x * 4, band, ib->x * n);
}
- imb_unpremultiply(dst, px, ib->x * ib->y);
- return 1;
+ return true;
+}
+
+int BK_imbuf_rgba(const BKImBuf *ib, int frame, unsigned char *dst)
+{
+ return ib && dst && frame >= 0 && frame < ib->frames && imb_rgba_rows(ib, frame, 0, ib->y, dst);
}
BKImBuf *BK_imbuf_from_rgba(const unsigned char *rgba, int w, int h, int stride)
diff --git a/kit/intern/image/loader.c b/kit/intern/image/loader.c
index 4dd6466..38c09bc 100644
--- a/kit/intern/image/loader.c
+++ b/kit/intern/image/loader.c
@@ -81,8 +81,25 @@ void bk_loader_notify(void)
/* What a request decodes (on a worker). */
static BKImBuf *decode(BKImRequest *req)
{
- if (req->anim) { /* the next frame of a playing animation */
- return imb_anim_next(req->anim, &req->delay, &req->wrapped);
+ if (req->anim) { /* the next frame of what plays: at the size now drawn, sought first */
+ imb_anim_retarget(req->anim, &req->target);
+ if (req->seek >= 0.0) {
+ imb_anim_seek(req->anim, req->seek);
+ }
+ BKImBuf *reuse = req->reuse;
+ req->reuse = NULL;
+ return imb_anim_next(req->anim, reuse, &req->pts, &req->dur, &req->wrapped);
+ }
+ if (req->movie) { /* a video: streamed from the file, the decoder kept */
+ BKImAnim *a = imb_movie_open(bk_image_source_path(req->source), &req->target);
+ BKImBuf *ib = a ? imb_anim_next(a, NULL, &req->pts, &req->dur, &req->wrapped) : NULL;
+ if (ib) {
+ req->anim = a;
+ }
+ else {
+ imb_anim_free(a);
+ }
+ return ib;
}
size_t size;
uint8_t *mem = imb_read_file(bk_image_source_path(req->source), 0, &size);
@@ -94,7 +111,7 @@ static BKImBuf *decode(BKImRequest *req)
/* an animation that plays: its first frame now, the decoder kept (it owns
* the file) for the next ones; one frame is just a still */
BKImAnim *a = imb_anim_open(mem, size, &req->target);
- BKImBuf *ib = a ? imb_anim_next(a, &req->delay, &req->wrapped) : NULL;
+ BKImBuf *ib = a ? imb_anim_next(a, NULL, &req->pts, &req->dur, &req->wrapped) : NULL;
if (ib && imb_anim_frames(a) > 1) {
req->anim = a;
}
diff --git a/kit/intern/image/movie.c b/kit/intern/image/movie.c
new file mode 100644
index 0000000..4054edc
--- /dev/null
+++ b/kit/intern/image/movie.c
@@ -0,0 +1,75 @@
+/* Movies: the video backends this build has (chosen at build time from the
+ * feature table, features.cmake: BUI_VIDEO_*), tried in order behind the one
+ * codec interface (BKImAnimCodec), and what a movie file looks like.
+ *
+ * movie_mf.c Windows Media Foundation (the OS decoders, DXVA)
+ * movie_avf.m macOS AVFoundation / VideoToolbox
+ * movie_gst.c Linux GStreamer (VA-API through its plugins)
+ * movie_dav1d.c AV1 with dav1d and a small MP4 / WebM demuxer (optional)
+ * movie_ffmpeg.c FFmpeg (optional)
+ *
+ * Every backend loads its system libraries at run time (LoadLibrary /
+ * dlopen / weak frameworks): a machine without them starts normally and
+ * just has no video.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <string.h>
+
+#include "image.h"
+
+bool imb_is_a_movie(const uint8_t *m, size_t n)
+{
+ static const uint8_t ebml[4] = {0x1A, 0x45, 0xDF, 0xA3}; /* WebM, Matroska */
+ static const uint8_t asf[4] = {0x30, 0x26, 0xB2, 0x75}; /* WMV, ASF */
+ static const uint8_t mpeg_ps[4] = {0x00, 0x00, 0x01, 0xBA}; /* MPEG program stream */
+ return (n >= 12 && memcmp(m + 4, "ftyp", 4) == 0 && memcmp(m + 8, "avif", 4) != 0 &&
+ memcmp(m + 8, "heic", 4) != 0) || /* MP4, MOV, M4V, 3GP (not AVIF/HEIC stills) */
+ (n >= 8 && (memcmp(m + 4, "moov", 4) == 0 || memcmp(m + 4, "mdat", 4) == 0 ||
+ memcmp(m + 4, "wide", 4) == 0 || memcmp(m + 4, "free", 4) == 0)) ||
+ (n >= 4 && (memcmp(m, ebml, 4) == 0 || memcmp(m, asf, 4) == 0 ||
+ memcmp(m, mpeg_ps, 4) == 0 || memcmp(m, "OggS", 4) == 0)) ||
+ (n >= 12 && memcmp(m, "RIFF", 4) == 0 && memcmp(m + 8, "AVI ", 4) == 0) ||
+ (n >= 3 && memcmp(m, "FLV", 3) == 0) || (n >= 1 && m[0] == 0x47 && n >= 64); /* TS */
+}
+
+const BKImAnimCodec *const imb_movie_codecs[] = {
+#ifdef BUI_VIDEO_DAV1D
+ &imb_movie_dav1d, /* AV1 in software, before the OS (which may lack AV1) */
+#endif
+#ifdef BUI_VIDEO_MF
+ &imb_movie_mf,
+#endif
+#ifdef BUI_VIDEO_AVF
+ &imb_movie_avf,
+#endif
+#ifdef BUI_VIDEO_GST
+ &imb_movie_gst,
+#endif
+#ifdef BUI_VIDEO_FFMPEG
+ &imb_movie_ffmpeg,
+#endif
+ NULL,
+};
+
+const char *BK_video_backends(void)
+{
+ static char names[96];
+ if (!names[0]) {
+ size_t n = 0;
+ for (int i = 0; imb_movie_codecs[i]; i++) {
+ const char *s = imb_movie_codecs[i]->name;
+ const size_t len = strlen(s);
+ if (n + len + 2 >= sizeof(names)) {
+ break;
+ }
+ if (n) {
+ names[n++] = ',';
+ }
+ memcpy(names + n, s, len);
+ n += len;
+ }
+ names[n] = '\0';
+ }
+ return names;
+}
diff --git a/kit/intern/image/movie_mf.c b/kit/intern/image/movie_mf.c
new file mode 100644
index 0000000..3cf32a6
--- /dev/null
+++ b/kit/intern/image/movie_mf.c
@@ -0,0 +1,420 @@
+/* Video on Windows: Media Foundation's source reader (the OS demuxers and
+ * decoders: H.264, HEVC, VP9 and AV1 where their extensions are installed,
+ * MPEG-4, WMV), with DXVA hardware decoding through a shared Direct3D 11
+ * device and the reader's video processor scaling to the size drawn on
+ * the GPU (MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING). Frames come
+ * back as NV12 at about the size drawn, copied into the frame anim.c gives.
+ *
+ * mfplat.dll, mfreadwrite.dll, d3d11.dll and ole32.dll are loaded on first
+ * use: Windows "N" editions (no Media Feature Pack) start normally and just
+ * have no video. The reader is free-threaded (COM MTA): one step at a time
+ * runs on whichever worker takes it.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifdef BUI_VIDEO_MF
+
+# define COBJMACROS
+# define WIN32_LEAN_AND_MEAN
+# include <windows.h>
+
+# include <initguid.h> /* the GUIDs below, defined here: nothing links mfuuid.lib */
+
+# include <d3d11.h>
+# include <mfapi.h>
+# include <mferror.h>
+# include <mfidl.h>
+# include <mfreadwrite.h>
+
+# include <string.h>
+
+# include "MEM_guardedalloc.h"
+
+# include "image.h"
+
+/* -------------------------------------------------------------------- */
+/** \name The system libraries, loaded once
+ * \{ */
+
+typedef HRESULT(WINAPI *MFStartupFn)(ULONG, DWORD);
+typedef HRESULT(WINAPI *MFCreateAttributesFn)(IMFAttributes **, UINT32);
+typedef HRESULT(WINAPI *MFCreateMediaTypeFn)(IMFMediaType **);
+typedef HRESULT(WINAPI *MFCreateDXGIDeviceManagerFn)(UINT *, IMFDXGIDeviceManager **);
+typedef HRESULT(WINAPI *MFCreateSourceReaderFromURLFn)(LPCWSTR, IMFAttributes *,
+ IMFSourceReader **);
+typedef HRESULT(WINAPI *CoInitializeExFn)(LPVOID, DWORD);
+typedef HRESULT(WINAPI *D3D11CreateDeviceFn)(IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT,
+ const D3D_FEATURE_LEVEL *, UINT, UINT,
+ ID3D11Device **, D3D_FEATURE_LEVEL *,
+ ID3D11DeviceContext **);
+
+static struct {
+ INIT_ONCE once;
+ bool ok;
+ MFCreateAttributesFn create_attributes;
+ MFCreateMediaTypeFn create_media_type;
+ MFCreateSourceReaderFromURLFn create_reader;
+ CoInitializeExFn co_init;
+ /* DXVA: one device for every reader (NULL: software decoding) */
+ IMFDXGIDeviceManager *manager;
+} MF = {INIT_ONCE_STATIC_INIT};
+
+static void *sym(HMODULE dll, const char *name)
+{
+ return dll ? (void *)GetProcAddress(dll, name) : NULL;
+}
+
+/* A video device for DXVA decoding and the reader's video processor. */
+static void dxva_init(HMODULE mfplat)
+{
+ MFCreateDXGIDeviceManagerFn create_manager = (MFCreateDXGIDeviceManagerFn)sym(
+ mfplat, "MFCreateDXGIDeviceManager");
+ D3D11CreateDeviceFn create_device = (D3D11CreateDeviceFn)sym(LoadLibraryW(L"d3d11.dll"),
+ "D3D11CreateDevice");
+ ID3D11Device *dev = NULL;
+ UINT token = 0;
+ if (!create_manager || !create_device ||
+ FAILED(create_device(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL,
+ D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT,
+ NULL, 0, D3D11_SDK_VERSION, &dev, NULL, NULL)))
+ {
+ return;
+ }
+ /* readers on several workers share it */
+ ID3D10Multithread *mt = NULL;
+ if (SUCCEEDED(ID3D11Device_QueryInterface(dev, &IID_ID3D10Multithread, (void **)&mt))) {
+ ID3D10Multithread_SetMultithreadProtected(mt, TRUE);
+ ID3D10Multithread_Release(mt);
+ }
+ if (SUCCEEDED(create_manager(&token, &MF.manager)) &&
+ FAILED(IMFDXGIDeviceManager_ResetDevice(MF.manager, (IUnknown *)dev, token)))
+ {
+ IMFDXGIDeviceManager_Release(MF.manager);
+ MF.manager = NULL;
+ }
+ ID3D11Device_Release(dev); /* the manager keeps it for the process */
+}
+
+static BOOL CALLBACK load(PINIT_ONCE once, PVOID param, PVOID *ctx)
+{
+ (void)once;
+ (void)param;
+ (void)ctx;
+ HMODULE mfplat = LoadLibraryW(L"mfplat.dll"), mfrw = LoadLibraryW(L"mfreadwrite.dll");
+ MFStartupFn startup = (MFStartupFn)sym(mfplat, "MFStartup");
+ MF.create_attributes = (MFCreateAttributesFn)sym(mfplat, "MFCreateAttributes");
+ MF.create_media_type = (MFCreateMediaTypeFn)sym(mfplat, "MFCreateMediaType");
+ MF.create_reader = (MFCreateSourceReaderFromURLFn)sym(mfrw, "MFCreateSourceReaderFromURL");
+ MF.co_init = (CoInitializeExFn)sym(LoadLibraryW(L"ole32.dll"), "CoInitializeEx");
+ if (!startup || !MF.create_attributes || !MF.create_media_type || !MF.create_reader ||
+ !MF.co_init)
+ {
+ return TRUE; /* no Media Foundation: no video, nothing else changes */
+ }
+ MF.co_init(NULL, COINIT_MULTITHREADED);
+ MF.ok = SUCCEEDED(startup(MF_VERSION, MFSTARTUP_LITE)); /* no network sources */
+ if (MF.ok) {
+ dxva_init(mfplat);
+ }
+ return TRUE;
+}
+
+/* Media Foundation, ready on this thread (COM MTA). */
+static bool mf_ready(void)
+{
+# ifdef _MSC_VER
+ static __declspec(thread) bool com;
+# else
+ static _Thread_local bool com;
+# endif
+ InitOnceExecuteOnce(&MF.once, load, NULL, NULL);
+ if (MF.ok && !com) {
+ const HRESULT hr = MF.co_init(NULL, COINIT_MULTITHREADED);
+ com = SUCCEEDED(hr) || hr == RPC_E_CHANGED_MODE; /* an STA host thread works too */
+ }
+ return MF.ok && com;
+}
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name The codec
+ * \{ */
+
+typedef struct MFMovie {
+ IMFSourceReader *reader;
+ int w, h; /* output frame (NV12) */
+ double frame_dur; /* from the frame rate, for samples without a duration */
+} MFMovie;
+
+# define VIDEO ((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM)
+
+static void mf_close(void *state)
+{
+ MFMovie *m = (MFMovie *)state;
+ if (m->reader) {
+ IMFSourceReader_Release(m->reader);
+ }
+ MEM_freeN(m);
+}
+
+/* The output's size, as the reader set it up. */
+static bool output_size(MFMovie *m)
+{
+ IMFMediaType *t = NULL;
+ UINT64 size = 0;
+ if (FAILED(IMFSourceReader_GetCurrentMediaType(m->reader, VIDEO, &t))) {
+ return false;
+ }
+ const bool ok = SUCCEEDED(IMFMediaType_GetUINT64(t, &MF_MT_FRAME_SIZE, &size));
+ IMFMediaType_Release(t);
+ m->w = (int)(size >> 32);
+ m->h = (int)(size & 0xFFFFFFFF);
+ return ok;
+}
+
+/* NV12 at out_w x out_h (the reader's video processor scales and converts). */
+static bool set_output(MFMovie *m, int out_w, int out_h, bool interlaced)
+{
+ IMFMediaType *t = NULL;
+ if (FAILED(MF.create_media_type(&t))) {
+ return false;
+ }
+ IMFMediaType_SetGUID(t, &MF_MT_MAJOR_TYPE, &MFMediaType_Video);
+ IMFMediaType_SetGUID(t, &MF_MT_SUBTYPE, &MFVideoFormat_NV12);
+ IMFMediaType_SetUINT64(t, &MF_MT_FRAME_SIZE, (UINT64)out_w << 32 | (UINT32)out_h);
+ IMFMediaType_SetUINT64(t, &MF_MT_PIXEL_ASPECT_RATIO, (UINT64)1 << 32 | 1);
+ if (interlaced) {
+ IMFMediaType_SetUINT32(t, &MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); /* deinterlace */
+ }
+ const HRESULT hr = IMFSourceReader_SetCurrentMediaType(m->reader, VIDEO, NULL, t);
+ IMFMediaType_Release(t);
+ return SUCCEEDED(hr) && output_size(m);
+}
+
+static wchar_t *wide_path(const char *path)
+{
+ const int n = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
+ wchar_t *w = n > 0 ? (wchar_t *)MEM_mallocN(sizeof(wchar_t) * (size_t)n, "movie path") : NULL;
+ if (w && !MultiByteToWideChar(CP_UTF8, 0, path, -1, w, n)) {
+ MEM_freeN(w);
+ w = NULL;
+ }
+ return w;
+}
+
+static IMFSourceReader *open_reader(const char *path)
+{
+ IMFAttributes *attr = NULL;
+ IMFSourceReader *reader = NULL;
+ wchar_t *wpath = wide_path(path);
+ if (wpath && SUCCEEDED(MF.create_attributes(&attr, 4))) {
+ IMFAttributes_SetUINT32(attr, &MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, TRUE);
+ IMFAttributes_SetUINT32(attr, &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE);
+ if (MF.manager) {
+ IMFAttributes_SetUnknown(attr, &MF_SOURCE_READER_D3D_MANAGER, (IUnknown *)MF.manager);
+ }
+ if (FAILED(MF.create_reader(wpath, attr, &reader))) {
+ reader = NULL;
+ }
+ IMFAttributes_Release(attr);
+ }
+ MEM_freeN(wpath);
+ return reader;
+}
+
+static void *mf_open(const char *path, const uint8_t *mem, size_t size, int want_w, int want_h,
+ BKImAnimInfo *r_info)
+{
+ (void)mem;
+ (void)size;
+ if (!path || !mf_ready()) {
+ return NULL;
+ }
+ MFMovie *m = (MFMovie *)MEM_callocN(sizeof(MFMovie), "MFMovie");
+ m->reader = open_reader(path);
+ IMFMediaType *native = NULL;
+ if (!m->reader ||
+ FAILED(IMFSourceReader_SetStreamSelection(m->reader, (DWORD)MF_SOURCE_READER_ALL_STREAMS,
+ FALSE)) ||
+ FAILED(IMFSourceReader_SetStreamSelection(m->reader, VIDEO, TRUE)) ||
+ FAILED(IMFSourceReader_GetNativeMediaType(m->reader, VIDEO, 0, &native)))
+ {
+ mf_close(m);
+ return NULL;
+ }
+ /* the intrinsic size: the coded frame through its pixel aspect ratio */
+ UINT64 fs = 0, par = 0, rate = 0;
+ UINT32 matrix = 0, interlace = MFVideoInterlace_Progressive;
+ IMFMediaType_GetUINT64(native, &MF_MT_FRAME_SIZE, &fs);
+ const bool has_par = SUCCEEDED(IMFMediaType_GetUINT64(native, &MF_MT_PIXEL_ASPECT_RATIO, &par));
+ IMFMediaType_GetUINT64(native, &MF_MT_FRAME_RATE, &rate);
+ IMFMediaType_GetUINT32(native, &MF_MT_YUV_MATRIX, &matrix);
+ IMFMediaType_GetUINT32(native, &MF_MT_INTERLACE_MODE, &interlace);
+ IMFMediaType_Release(native);
+ int src_w = (int)(fs >> 32), src_h = (int)(fs & 0xFFFFFFFF);
+ const UINT32 pn = (UINT32)(par >> 32), pd = (UINT32)par;
+ if (has_par && pn && pd && pn != pd) {
+ src_w = (int)((double)src_w * pn / pd + 0.5);
+ }
+ if (src_w <= 0 || src_h <= 0 || src_w > BK_IMB_MAX_SIDE || src_h > BK_IMB_MAX_SIDE) {
+ mf_close(m);
+ return NULL;
+ }
+ /* decode at the size asked (never above the source), even for NV12 */
+ int out_w = want_w > 0 ? BK_MIN(want_w, src_w) : src_w;
+ int out_h = want_h > 0 ? BK_MIN(want_h, src_h) : src_h;
+ out_w = BK_MAX(2, out_w & ~1);
+ out_h = BK_MAX(2, out_h & ~1);
+ const bool interlaced = interlace != MFVideoInterlace_Progressive;
+ if (!set_output(m, out_w, out_h, interlaced) &&
+ !set_output(m, BK_MAX(2, src_w & ~1), BK_MAX(2, src_h & ~1), interlaced))
+ {
+ mf_close(m); /* no NV12 output: no decoder for this format */
+ return NULL;
+ }
+ const UINT32 rn = (UINT32)(rate >> 32), rd = (UINT32)rate;
+ m->frame_dur = rn && rd ? (double)rd / rn : 1.0 / 30.0;
+ PROPVARIANT dur;
+ memset(&dur, 0, sizeof(dur)); /* VT_UI8: nothing to clear */
+ r_info->duration =
+ SUCCEEDED(IMFSourceReader_GetPresentationAttribute(
+ m->reader, (DWORD)MF_SOURCE_READER_MEDIASOURCE, &MF_PD_DURATION, &dur)) &&
+ dur.vt == VT_UI8 ?
+ (double)dur.uhVal.QuadPart / 1e7 :
+ 0.0;
+ r_info->w = m->w;
+ r_info->h = m->h;
+ r_info->src_w = src_w;
+ r_info->src_h = src_h;
+ r_info->loops = 1;
+ r_info->format = matrix == MFVideoTransferMatrix_BT601 ? BUI_PIXMAP_NV12_BT601 :
+ matrix == MFVideoTransferMatrix_BT709 ? BUI_PIXMAP_NV12_BT709 :
+ src_h >= 720 ? BUI_PIXMAP_NV12_BT709 :
+ BUI_PIXMAP_NV12_BT601;
+ return m;
+}
+
+/* The frame's planes into `dst` (NV12, its size). */
+static bool copy_frame(MFMovie *m, IMFSample *sample, BKImBuf *dst)
+{
+ IMFMediaBuffer *buf = NULL;
+ if (FAILED(IMFSample_GetBufferByIndex(sample, 0, &buf))) {
+ return false;
+ }
+ /* rows of a 2D buffer are `pitch` apart; chroma starts after the plane's
+ * allocated height (a GPU surface may be taller than the frame) */
+ IMF2DBuffer *b2 = NULL;
+ IMFDXGIBuffer *dx = NULL;
+ BYTE *scan0 = NULL;
+ LONG pitch = 0;
+ int plane_h = m->h;
+ bool locked2d = false;
+ if (SUCCEEDED(IMFMediaBuffer_QueryInterface(buf, &IID_IMFDXGIBuffer, (void **)&dx))) {
+ ID3D11Texture2D *tex = NULL;
+ if (SUCCEEDED(IMFDXGIBuffer_GetResource(dx, &IID_ID3D11Texture2D, (void **)&tex))) {
+ D3D11_TEXTURE2D_DESC td;
+ ID3D11Texture2D_GetDesc(tex, &td);
+ plane_h = (int)td.Height;
+ ID3D11Texture2D_Release(tex);
+ }
+ IMFDXGIBuffer_Release(dx);
+ }
+ if (SUCCEEDED(IMFMediaBuffer_QueryInterface(buf, &IID_IMF2DBuffer, (void **)&b2))) {
+ locked2d = SUCCEEDED(IMF2DBuffer_Lock2D(b2, &scan0, &pitch));
+ }
+ DWORD len = 0;
+ if (!locked2d && SUCCEEDED(IMFMediaBuffer_Lock(buf, &scan0, NULL, &len))) {
+ pitch = m->w; /* a contiguous buffer: packed rows */
+ }
+ const bool ok = scan0 && pitch >= dst->x && plane_h >= dst->y;
+ if (ok) {
+ uint8_t *y = (uint8_t *)dst->rect, *uv = y + (size_t)dst->x * dst->y;
+ const BYTE *sy = scan0, *suv = scan0 + (size_t)pitch * plane_h;
+ for (int r = 0; r < dst->y; r++) {
+ memcpy(y + (size_t)r * dst->x, sy + (size_t)r * pitch, (size_t)dst->x);
+ }
+ for (int r = 0; r < dst->y / 2; r++) {
+ memcpy(uv + (size_t)r * dst->x, suv + (size_t)r * pitch, (size_t)dst->x);
+ }
+ }
+ if (locked2d) {
+ IMF2DBuffer_Unlock2D(b2);
+ }
+ else if (scan0) {
+ IMFMediaBuffer_Unlock(buf);
+ }
+ if (b2) {
+ IMF2DBuffer_Release(b2);
+ }
+ IMFMediaBuffer_Release(buf);
+ return ok;
+}
+
+static int mf_next(void *state, BKImBuf *dst, double *r_pts, double *r_dur)
+{
+ MFMovie *m = (MFMovie *)state;
+ for (int gaps = 0; gaps < 64; gaps++) {
+ DWORD stream = 0, flags = 0;
+ LONGLONG ts = 0;
+ IMFSample *sample = NULL;
+ if (FAILED(IMFSourceReader_ReadSample(m->reader, VIDEO, 0, &stream, &flags, &ts, &sample))) {
+ return -1;
+ }
+ if (flags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED) {
+ const int w = m->w, h = m->h;
+ if (!output_size(m) || m->w != w || m->h != h) {
+ if (sample) {
+ IMFSample_Release(sample);
+ }
+ return -1; /* resized mid-stream: anim.c reopens at its size (a seek) */
+ }
+ }
+ if (flags & (MF_SOURCE_READERF_ENDOFSTREAM | MF_SOURCE_READERF_ERROR)) {
+ if (sample) {
+ IMFSample_Release(sample);
+ }
+ return (flags & MF_SOURCE_READERF_ERROR) ? -1 : 0;
+ }
+ if (!sample) {
+ continue; /* a stream tick (gap): read on */
+ }
+ LONGLONG d = 0;
+ const bool has_dur = SUCCEEDED(IMFSample_GetSampleDuration(sample, &d)) && d > 0;
+ const bool ok = copy_frame(m, sample, dst);
+ IMFSample_Release(sample);
+ if (!ok) {
+ return -1;
+ }
+ *r_pts = (double)ts / 1e7;
+ *r_dur = has_dur ? (double)d / 1e7 : m->frame_dur;
+ return 1;
+ }
+ return -1;
+}
+
+static bool mf_seek(void *state, double t)
+{
+ MFMovie *m = (MFMovie *)state;
+ PROPVARIANT pos;
+ memset(&pos, 0, sizeof(pos));
+ pos.vt = VT_I8;
+ pos.hVal.QuadPart = (LONGLONG)(t * 1e7);
+ static const GUID time_format = {0}; /* GUID_NULL: 100 ns units */
+ return SUCCEEDED(IMFSourceReader_SetCurrentPosition(m->reader, &time_format, &pos));
+}
+
+static size_t mf_bytes(const void *state)
+{
+ const MFMovie *m = (const MFMovie *)state;
+ /* what this process holds for it: the reader's output samples (an estimate;
+ * decoder surfaces live in video memory) */
+ return sizeof(*m) + (size_t)m->w * m->h * 3;
+}
+
+const BKImAnimCodec imb_movie_mf = {
+ "mf", true, imb_is_a_movie, mf_open, mf_next, mf_seek, mf_bytes, mf_close};
+
+/** \} */
+
+#endif /* BUI_VIDEO_MF */
diff --git a/kit/intern/image/play.c b/kit/intern/image/play.c
index 5d85688..d8ab3cb 100644
--- a/kit/intern/image/play.c
+++ b/kit/intern/image/play.c
@@ -1,36 +1,59 @@
-/* Playing an animation held by a cache entry: the frame clock.
+/* The one clock of everything that plays: GIF, animated WebP and video,
+ * owned by a cache entry (a shared <img>, or a <video>'s private entry).
*
* The entry shows one frame; this keeps the decoder and the next frame,
* decoded ahead on the workers (one step request at a time), and says when
- * the next frame is due (each frame's own delay, GIF and WebP play counts).
- * It only moves on when the frame on screen was drawn: a window that isn't
- * rendering (hidden, minimized) costs one decode, then nothing, and an
- * entry nobody shows is not stepped at all (cache.c).
+ * the next frame is due (each frame's own duration over the playback rate,
+ * GIF and WebP play counts, a video's loop). It only moves on when the frame
+ * on screen was drawn: a window that isn't rendering (hidden, minimized)
+ * costs one decode, then nothing, and an entry nobody shows is not stepped
+ * at all (cache.c). Movies decode into the frame the screen no longer shows
+ * (two frames, no allocation per frame).
+ *
+ * Controls (pause, seek, rate, loop) are the <video> element's; a seek's
+ * frame shows as soon as it is decoded, paused or not.
*
* SPDX-License-Identifier: GPL-2.0-or-later */
+#include <float.h>
+
#include "MEM_guardedalloc.h"
#include "image.h"
struct BKImPlay {
- BKImAnim *anim; /* NULL while a step request has it */
+ BKImAnim *anim; /* NULL before the first frame and while a step request has it */
BKImBuf *next; /* the frame after the one shown (NULL while decoding) */
- int next_delay;
+ double next_pts, next_dur;
bool next_wrapped; /* `next` starts another play */
- int delay; /* how long the shown frame stays (ms) */
+ bool next_seek; /* `next` is a seek's frame: shown at once */
+ BKImBuf *spare; /* movies: the frame the screen let go of, decoded into next */
+ double pts, dur; /* the shown frame's time and duration (seconds) */
double due; /* when it is over (bk_time seconds; 0: not timed yet) */
- int plays, loops; /* plays finished; loops 0 = forever */
- bool drawn; /* the shown frame reached the screen */
- bool finished; /* the last play's last frame is showing */
+ int plays; /* plays finished */
+ int loops; /* the play count (0 = forever): the file's, or set */
+ int loops_set; /* -1: the file's */
+ int file_loops, src_w, src_h;
+ double duration; /* seconds, 0 when unknown */
+ double rate;
+ double seek; /* a seek to request (< 0: none) */
+ double seek_t; /* the last seek's time, while its frame isn't shown */
+ bool seeking;
+ unsigned seeks; /* seeks asked for; a step taken before the last one is stale */
+ unsigned step_seeks; /* `seeks` when the step out was taken */
+ bool step_seek; /* the step out seeks */
+ bool loaded, failed, paused, drawn;
+ bool finished; /* the last play's last frame shows */
};
-BKImPlay *imb_play_new(BKImAnim *anim, int delay)
+BKImPlay *imb_play_new(bool paused, int loops)
{
BKImPlay *p = (BKImPlay *)MEM_callocN(sizeof(BKImPlay), "BKImPlay");
- p->anim = anim;
- p->delay = delay;
- p->loops = imb_anim_loops(anim);
+ p->paused = paused;
+ p->loops_set = loops;
+ p->loops = BK_MAX(0, loops);
+ p->rate = 1.0;
+ p->seek = -1.0;
return p;
}
@@ -39,77 +62,199 @@ void imb_play_free(BKImPlay *p)
if (p) {
imb_anim_free(p->anim);
BK_imbuf_free(p->next);
+ BK_imbuf_free(p->spare);
MEM_freeN(p);
}
}
size_t imb_play_bytes(const BKImPlay *p)
{
- return sizeof(*p) + (p->anim ? imb_anim_bytes(p->anim) : 0) + (p->next ? p->next->size : 0);
+ return sizeof(*p) + (p->anim ? imb_anim_bytes(p->anim) : 0) + (p->next ? p->next->size : 0) +
+ (p->spare ? p->spare->size : 0);
+}
+
+void imb_play_start(BKImPlay *p, BKImAnim *anim, double pts, double dur)
+{
+ p->anim = anim;
+ p->loaded = true;
+ p->pts = pts;
+ p->dur = dur;
+ const BKImAnimInfo *info = imb_anim_info(anim);
+ p->file_loops = info->loops;
+ p->src_w = info->src_w;
+ p->src_h = info->src_h;
+ p->duration = info->duration;
+ if (p->loops_set < 0) {
+ p->loops = p->file_loops;
+ }
+}
+
+void imb_play_fail(BKImPlay *p)
+{
+ p->failed = true;
+}
+
+void imb_play_recycle(BKImPlay *p, BKImBuf *old)
+{
+ if (old && old->stream && !p->spare) {
+ p->spare = old; /* the next decode goes into it */
+ }
+ else {
+ BK_imbuf_free(old);
+ }
}
-BKImAnim *imb_play_take_step(BKImPlay *p)
+BKImAnim *imb_play_take_step(BKImPlay *p, BKImBuf **r_reuse, double *r_seek)
{
- if (!p->anim || p->next || p->finished) {
+ if (!p->anim || p->failed || (p->seek < 0.0 && (p->next || p->finished))) {
return NULL;
}
BKImAnim *a = p->anim;
p->anim = NULL;
+ *r_reuse = p->spare;
+ p->spare = NULL;
+ *r_seek = p->seek;
+ p->step_seek = p->seek >= 0.0;
+ p->step_seeks = p->seeks;
+ p->seek = -1.0;
return a;
}
-void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, int delay, bool wrapped)
+void imb_play_step_done(BKImPlay *p, BKImAnim *anim, BKImBuf *next, double pts, double dur,
+ bool wrapped)
{
p->anim = anim;
- BK_imbuf_free(p->next);
- p->next = next;
- p->next_delay = delay;
- p->next_wrapped = wrapped;
+ if (p->step_seeks != p->seeks) {
+ imb_play_recycle(p, next); /* decoded before a seek: stale */
+ return;
+ }
if (!next) {
p->finished = true; /* a decoding error: stay on this frame */
+ return;
}
+ imb_play_recycle(p, p->next);
+ p->next = next;
+ p->next_pts = pts;
+ p->next_dur = dur;
+ p->next_wrapped = wrapped;
+ p->next_seek = p->step_seek;
}
-void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim)
+void imb_play_step_cancelled(BKImPlay *p, BKImAnim *anim, BKImBuf *reuse)
{
p->anim = anim; /* stepped again when it is wanted */
+ imb_play_recycle(p, reuse);
+ if (p->step_seek && p->step_seeks == p->seeks && p->seek < 0.0) {
+ p->seek = p->seek_t; /* the seek is still to do */
+ }
}
void imb_play_drawn(BKImPlay *p, double now)
{
p->drawn = true;
if (p->due == 0.0) {
- p->due = now + p->delay / 1000.0; /* the clock starts when it first shows */
+ p->due = now + p->dur / p->rate; /* the clock starts when it first shows */
}
}
-void imb_play_rewind_clock(BKImPlay *p)
-{
- p->due = 0.0; /* resumed: the frame gets its whole delay again */
-}
-
double imb_play_due(const BKImPlay *p)
{
- return p->drawn && !p->finished && p->next ? p->due : 0.0;
+ if (p->next && p->next_seek) {
+ return DBL_MIN; /* now */
+ }
+ return p->drawn && !p->finished && !p->paused && p->next ? p->due : 0.0;
}
BKImBuf *imb_play_advance(BKImPlay *p, double now)
{
- if (imb_play_due(p) == 0.0 || now < p->due) {
+ const double due = imb_play_due(p);
+ if (due == 0.0 || now < due) {
return NULL;
}
- if (p->next_wrapped && p->loops > 0 && p->plays + 1 >= p->loops) {
+ if (!p->next_seek && p->next_wrapped && p->loops > 0 && p->plays + 1 >= p->loops) {
p->finished = true; /* the last frame of the last play stays */
- BK_imbuf_free(p->next);
+ imb_play_recycle(p, p->next);
p->next = NULL;
return NULL;
}
BKImBuf *frame = p->next;
p->next = NULL;
- p->plays += p->next_wrapped;
- p->delay = p->next_delay;
- /* keep the beat, unless far behind (paused, slow decodes): restart it */
- p->due = now - p->due > 0.25 ? now + p->delay / 1000.0 : p->due + p->delay / 1000.0;
+ p->pts = p->next_pts;
+ p->dur = p->next_dur;
+ if (p->next_seek) {
+ p->due = 0.0; /* timed from when it shows */
+ p->next_seek = false;
+ p->seeking = false;
+ }
+ else {
+ p->plays += p->next_wrapped;
+ /* keep the beat, unless far behind (paused, slow decodes): restart it */
+ p->due = now - p->due > 0.25 ? now + p->dur / p->rate : p->due + p->dur / p->rate;
+ }
p->drawn = false;
return frame;
}
+
+void imb_play_pause(BKImPlay *p, bool paused)
+{
+ if (p->paused && !paused) {
+ p->due = 0.0; /* resumed: the frame gets its time again */
+ if (p->finished) { /* playing an ended video starts it over */
+ imb_play_seek(p, 0.0);
+ }
+ }
+ p->paused = paused;
+}
+
+void imb_play_seek(BKImPlay *p, double t)
+{
+ const double duration = imb_play_duration(p);
+ p->seek = p->seek_t = BK_CLAMP(t, 0.0, duration > 0.0 ? duration : t);
+ p->seeking = true;
+ p->seeks++;
+ p->finished = false;
+ p->plays = 0;
+ imb_play_recycle(p, p->next); /* decoded before the seek */
+ p->next = NULL;
+ p->next_seek = false;
+}
+
+void imb_play_set_rate(BKImPlay *p, double rate)
+{
+ p->rate = BK_CLAMP(rate, 1.0 / 16.0, 16.0);
+}
+
+void imb_play_set_loops(BKImPlay *p, int loops)
+{
+ p->loops_set = loops;
+ p->loops = loops >= 0 ? loops : p->file_loops;
+ if (p->loops == 0) {
+ p->finished = false; /* an ended video looping again goes on from the top */
+ }
+}
+
+BKImPlayState imb_play_state(const BKImPlay *p)
+{
+ return p->failed ? BK_PLAY_ERROR :
+ !p->loaded ? BK_PLAY_LOADING :
+ p->finished ? BK_PLAY_ENDED :
+ p->paused ? BK_PLAY_PAUSED :
+ BK_PLAY_PLAYING;
+}
+
+double imb_play_time(const BKImPlay *p)
+{
+ return p->seeking ? p->seek_t : p->pts;
+}
+
+double imb_play_duration(const BKImPlay *p)
+{
+ return p->duration;
+}
+
+bool imb_play_size(const BKImPlay *p, int *r_w, int *r_h)
+{
+ *r_w = p->src_w;
+ *r_h = p->src_h;
+ return p->loaded;
+}
diff --git a/kit/intern/image/video.c b/kit/intern/image/video.c
new file mode 100644
index 0000000..c8463a2
--- /dev/null
+++ b/kit/intern/image/video.c
@@ -0,0 +1,361 @@
+/* <video>: a private cache entry (cache.c: the same requests, workers and
+ * visibility tracking as images) with its own clock (play.c: pause, seek,
+ * rate, loop), shown on "video" and "image" nodes and canvas items, or
+ * copied into a texture for Dear ImGui. The "video" node draws Blender-style
+ * controls over it: a play/pause button and a scrubber with the time, like
+ * the Timeline's playback controls and frame slider.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "MEM_guardedalloc.h"
+
+#include "image.h"
+#include "kit_intern.h"
+
+struct BKVideo {
+ BKImEntry *entry; /* frames and clock */
+ BKImageRef poster; /* shown until the first frame */
+ bool controls;
+ bool scrubbing; /* the scrubber is dragged (the video was playing: `resume`) */
+ bool resume;
+ void (*fn)(BKVideo *v, int state, void *data);
+ void *data;
+ uint64_t tex_key; /* BK_video_texture: the frame last copied */
+};
+
+static BKImPlay *play_of(const BKVideo *v)
+{
+ return bk_image_entry_play(v->entry);
+}
+
+static void heard(void *owner)
+{
+ BKVideo *v = (BKVideo *)owner;
+ if (v->fn) {
+ v->fn(v, BK_video_state(v), v->data);
+ }
+}
+
+BKVideo *BK_video_new(const char *src)
+{
+ BKImageSource *s = bk_image_source_get(src);
+ if (!s) {
+ return NULL;
+ }
+ BKVideo *v = (BKVideo *)MEM_callocN(sizeof(BKVideo), "BKVideo");
+ /* muted autoplay, played once (HTML: loop off) */
+ v->entry = bk_image_entry_new(s, imb_play_new(false, 1), heard, v);
+ bk_image_source_release(s); /* the entry keeps it */
+ return v;
+}
+
+void BK_video_free(BKVideo *v)
+{
+ if (v) {
+ bk_image_entry_free(v->entry);
+ bk_image_ref_clear(&v->poster);
+ MEM_freeN(v);
+ }
+}
+
+/* A control changed the clock: decode and redraw now, tell the listener. */
+static void changed(BKVideo *v)
+{
+ bk_image_entry_changed(v->entry);
+}
+
+void BK_video_play(BKVideo *v)
+{
+ if (v) {
+ imb_play_pause(play_of(v), false);
+ changed(v);
+ }
+}
+
+void BK_video_pause(BKVideo *v)
+{
+ if (v) {
+ imb_play_pause(play_of(v), true);
+ changed(v);
+ }
+}
+
+void BK_video_seek(BKVideo *v, double seconds)
+{
+ if (v && isfinite(seconds)) {
+ imb_play_seek(play_of(v), seconds);
+ changed(v);
+ }
+}
+
+void BK_video_set_rate(BKVideo *v, double rate)
+{
+ if (v && rate > 0.0 && isfinite(rate)) {
+ imb_play_set_rate(play_of(v), rate);
+ }
+}
+
+void BK_video_set_loop(BKVideo *v, int loop)
+{
+ if (v) {
+ imb_play_set_loops(play_of(v), loop ? 0 : 1);
+ changed(v);
+ }
+}
+
+void BK_video_set_poster(BKVideo *v, const char *src)
+{
+ if (v) {
+ if (src && src[0]) {
+ bk_image_ref_set_source(&v->poster, src);
+ }
+ else {
+ bk_image_ref_clear(&v->poster);
+ }
+ }
+}
+
+void BK_video_set_controls(BKVideo *v, int on)
+{
+ if (v) {
+ v->controls = on != 0;
+ changed(v);
+ }
+}
+
+int BK_video_state(const BKVideo *v)
+{
+ return v ? (int)imb_play_state(play_of(v)) : BK_VIDEO_ERROR;
+}
+
+double BK_video_time(const BKVideo *v)
+{
+ return v ? imb_play_time(play_of(v)) : 0.0;
+}
+
+double BK_video_duration(const BKVideo *v)
+{
+ return v ? imb_play_duration(play_of(v)) : 0.0;
+}
+
+int BK_video_size(const BKVideo *v, int *r_w, int *r_h)
+{
+ int w = 0, h = 0;
+ const bool ok = v && bk_video_size(v, &w, &h);
+ if (r_w) {
+ *r_w = w;
+ }
+ if (r_h) {
+ *r_h = h;
+ }
+ return ok;
+}
+
+void BK_video_set_listener(BKVideo *v, void (*fn)(BKVideo *v, int state, void *data), void *data)
+{
+ if (v) {
+ v->fn = fn;
+ v->data = data;
+ }
+}
+
+bool bk_video_size(const BKVideo *v, int *r_w, int *r_h)
+{
+ return imb_play_size(play_of(v), r_w, r_h) || bk_image_ref_size(&v->poster, r_w, r_h);
+}
+
+const BKImBuf *bk_video_frame(BKVideo *v, const BKImageFit *target, const int rect[4])
+{
+ const BKImBuf *ib = bk_image_entry_get(v->entry, target, rect);
+ if (!ib && v->poster.kind == BK_IMREF_SOURCE) {
+ ib = bk_image_cache_get(v->poster.u.source, target, rect);
+ }
+ return ib;
+}
+
+int BK_video_texture(BKVideo *v, int tex, int max_w, int max_h)
+{
+ if (!v) {
+ return 0;
+ }
+ const BKImageFit target = imb_fit_within(max_w, max_h);
+ const BKImBuf *ib = bk_image_entry_get(v->entry, &target, NULL); /* shown by the host */
+ if (!ib || ib->format == BUI_PIXMAP_ARGB) {
+ return ib && !tex ? BK_imbuf_texture(ib, 0) : tex; /* the poster, or nothing yet */
+ }
+ int tw = 0, th = 0;
+ if (tex && (!BUI_texture_size(tex, &tw, &th) || tw != ib->x || th != ib->y)) {
+ BUI_texture_free(tex); /* resized: a new texture */
+ tex = 0;
+ }
+ if (!tex) {
+ tex = BK_imbuf_texture(ib, 0);
+ }
+ else if (ib->key != v->tex_key) {
+ BK_imbuf_texture_update(ib, 0, tex); /* in place */
+ }
+ v->tex_key = tex ? ib->key : 0;
+ return tex;
+}
+
+/* -------------------------------------------------------------------- */
+/** \name The "video" node: the frame, fitted, and Blender-style controls
+ * \{ */
+
+void BK_node_set_video(BKNode *n, BKVideo *v, int fit)
+{
+ bk_image_ref_set_video(&n->image, v);
+ n->image_fit = fit;
+ bk_relayout(n);
+ bk_invalidate(n);
+}
+
+static BKVideo *node_video(const BKNode *n)
+{
+ return n->image.kind == BK_IMREF_VIDEO ? n->image.u.video : NULL;
+}
+
+/* The controls bar along the bottom: the play button, then the scrubber. */
+static bool controls_rects(const BKNode *n, BKRect *r_button, BKRect *r_scrub)
+{
+ const BKVideo *v = node_video(n);
+ const int uu = bk_unit(), pad = BK_MAX(2, uu / 5);
+ if (!v || !v->controls || n->h < uu + 2 * pad || n->w < 3 * uu) {
+ return false;
+ }
+ const int y = n->y + n->h - uu - pad;
+ *r_button = bk_rect(n->x + pad, y, uu, uu);
+ *r_scrub = bk_rect(n->x + 2 * pad + uu, y, n->w - 3 * pad - uu, uu);
+ return true;
+}
+
+static void video_measure(BKNode *n, int *rw, int *rh)
+{
+ if (!bk_node_image_size(n, rw, rh)) { /* HTML's default, until the size is known */
+ *rw = (int)(300 * bk_px());
+ *rh = (int)(150 * bk_px());
+ }
+}
+
+static void format_time(char *buf, size_t size, double t)
+{
+ const int s = (int)BK_MAX(0.0, t);
+ snprintf(buf, size, "%d:%02d", s / 60, s % 60);
+}
+
+static void video_draw(BKNode *n)
+{
+ const float box[4] = {(float)n->x, (float)n->y, (float)n->w, (float)n->h};
+ bk_image_ref_draw(&n->image, n->image_fit, 0, box, NODE_ENABLED(n) ? 1.0f : 0.5f);
+ BKRect rb, rs;
+ if (!controls_rects(n, &rb, &rs)) {
+ return;
+ }
+ const BKVideo *v = node_video(n);
+ const int state = BK_video_state(v);
+ BKWCol w;
+ bk_widget_state(&w, &bk_theme.regular, NODE_ENABLED(n) ? 0 : ST_DISABLED);
+ bk_widget_but(&w, &rb, 0, CNR_ALL);
+ const bool playing = state == BK_VIDEO_PLAYING || (v->scrubbing && v->resume);
+ bk_icon(bk_icons_lookup(playing ? "PAUSE" : "PLAY"), rb.xmin + bk_rw(&rb) * 0.5f,
+ rb.ymin + bk_rh(&rb) * 0.5f, w.text, 1.0f, 0);
+ /* the scrubber: Blender's number slider, filled to the time */
+ const double dur = BK_video_duration(v), t = BK_video_time(v);
+ bk_widget_state(&w, &bk_theme.numslider, v->scrubbing ? ST_SELECT : 0);
+ bk_widget_numslider(&w, &rs, 0, CNR_ALL, dur > 0.0 ? (float)BK_CLAMP(t / dur, 0.0, 1.0) : 0.0f);
+ char a[16], b[16], label[40];
+ format_time(a, sizeof(a), t);
+ format_time(b, sizeof(b), dur);
+ snprintf(label, sizeof(label), "%s / %s", a, b);
+ bk_text_in(&rs, label, w.text, BK_ALIGN_CENTER, false);
+}
+
+static void scrub_to(BKNode *n, int x)
+{
+ BKRect rb, rs;
+ BKVideo *v = node_video(n);
+ const double dur = BK_video_duration(v);
+ if (controls_rects(n, &rb, &rs) && dur > 0.0) {
+ const double f = (x - rs.xmin) / (double)BK_MAX(1, bk_rw(&rs));
+ BK_video_seek(v, BK_CLAMP(f, 0.0, 1.0) * dur);
+ }
+}
+
+static int video_press(BKNode *n, BKEvent *e)
+{
+ BKRect rb, rs;
+ BKVideo *v = node_video(n);
+ if (!controls_rects(n, &rb, &rs) || !NODE_ENABLED(n)) {
+ return 0;
+ }
+ if (bk_rect_inside(&rb, e->x, e->y)) {
+ if (BK_video_state(v) == BK_VIDEO_PLAYING) {
+ BK_video_pause(v);
+ }
+ else {
+ BK_video_play(v);
+ }
+ return 1;
+ }
+ if (bk_rect_inside(&rs, e->x, e->y)) {
+ /* scrubbing holds the video, like dragging Blender's frame slider */
+ v->resume = BK_video_state(v) == BK_VIDEO_PLAYING;
+ v->scrubbing = true;
+ BK_video_pause(v);
+ scrub_to(n, e->x);
+ return 1;
+ }
+ return 0;
+}
+
+static int video_drag(BKNode *n, BKEvent *e)
+{
+ BKVideo *v = node_video(n);
+ if (v && v->scrubbing) {
+ scrub_to(n, e->x);
+ }
+ return 1;
+}
+
+static int video_release(BKNode *n, BKEvent *e)
+{
+ (void)e;
+ BKVideo *v = node_video(n);
+ if (v && v->scrubbing) {
+ v->scrubbing = false;
+ if (v->resume) {
+ BK_video_play(v);
+ }
+ }
+ return 1;
+}
+
+static int video_key(BKNode *n, BKEvent *e)
+{
+ BKVideo *v = node_video(n);
+ if (!v || !v->controls || !bk_keysym_is(e->keysym, "space")) {
+ return 0;
+ }
+ if (BK_video_state(v) == BK_VIDEO_PLAYING) {
+ BK_video_pause(v);
+ }
+ else {
+ BK_video_play(v);
+ }
+ return 1;
+}
+
+const BKClass bk_cls_video = {.name = "video",
+ .flags = CLS_INTERACTIVE,
+ .measure = video_measure,
+ .draw = video_draw,
+ .press = video_press,
+ .drag = video_drag,
+ .release = video_release,
+ .key = video_key};
+
+/** \} */
diff --git a/kit/intern/kit_core.c b/kit/intern/kit_core.c
index c83176b..275af1f 100644
--- a/kit/intern/kit_core.c
+++ b/kit/intern/kit_core.c
@@ -158,7 +158,7 @@ static const BKClass *const g_classes[] = {
&bk_cls_menubutton, &bk_cls_menubar, &bk_cls_list, &bk_cls_tree,
&bk_cls_gridview, &bk_cls_ramp, &bk_cls_curve, &bk_cls_normal,
&bk_cls_histogram, &bk_cls_frame, &bk_cls_radio, &bk_cls_canvas,
- &bk_cls_textarea, &bk_cls_image, &bk_cls_meter, &bk_cls_custom,
+ &bk_cls_textarea, &bk_cls_image, &bk_cls_meter, &bk_cls_custom, &bk_cls_video,
&bk_cls_filebrowser,
};
diff --git a/kit/intern/kit_frame.c b/kit/intern/kit_frame.c
index 600c8bd..abbf005 100644
--- a/kit/intern/kit_frame.c
+++ b/kit/intern/kit_frame.c
@@ -824,10 +824,14 @@ static void citem_assign(CItem *c, const BKCanvasItem *item)
memcpy(c->text, item->text, (size_t)tl);
}
/* one pixel source per image item (validated here, at the boundary) */
- const int sources = (item->rgba != NULL) + (item->imbuf != NULL) + (item->src != NULL);
+ const int sources = (item->rgba != NULL) + (item->imbuf != NULL) + (item->src != NULL) +
+ (item->video != NULL);
if (item->type != BK_ITEM_IMAGE || sources != 1) {
bk_image_ref_clear(&c->img);
}
+ else if (item->video) {
+ bk_image_ref_set_video(&c->img, item->video);
+ }
else if (item->imbuf) {
bk_image_ref_set_imbuf(&c->img, item->imbuf);
}
@@ -843,6 +847,7 @@ static void citem_assign(CItem *c, const BKCanvasItem *item)
c->it.rgba = NULL;
c->it.src = NULL;
c->it.imbuf = NULL;
+ c->it.video = NULL;
}
/* An image item's box in canvas space: x0 y0 x1 y1 with 4 coordinates,
diff --git a/kit/intern/kit_intern.h b/kit/intern/kit_intern.h
index b20b319..1a25136 100644
--- a/kit/intern/kit_intern.h
+++ b/kit/intern/kit_intern.h
@@ -31,6 +31,11 @@ static inline BKRect bk_rect(int x, int y, int w, int h)
BKRect r = {x, y, x + w, y + h};
return r;
}
+/* BLI_rcti_isect_pt */
+static inline bool bk_rect_inside(const BKRect *r, int x, int y)
+{
+ return x >= r->xmin && x < r->xmax && y >= r->ymin && y < r->ymax;
+}
/* Corner flags (Blender's CNR_*; "top" = screen top). */
enum { CNR_TL = 1, CNR_TR = 2, CNR_BR = 4, CNR_BL = 8, CNR_ALL = 15 };
@@ -551,6 +556,7 @@ void bk_theme_env_init(void);
/* Custom nodes (kit_custom.c). The host timer that wakes them. */
#define BK_TIMER_CUSTOM (-1)
extern const BKClass bk_cls_custom;
+extern const BKClass bk_cls_video; /* image/video.c */
#define BK_IS_CUSTOM(n) ((n) && (n)->cls == &bk_cls_custom)
/* Delivers an event (fills x, y relative to the node); returns its result. */
int bk_custom_event(BKNode *n, BKCustomEvent *e, int wx, int wy);
diff --git a/tests/video_test.c b/tests/video_test.c
new file mode 100644
index 0000000..9129385
--- /dev/null
+++ b/tests/video_test.c
@@ -0,0 +1,353 @@
+/* Video: load, play, pause, seek, rate, loop and end through the public API
+ * (BK_video_*), drawn on a canvas like an app would; nothing decoded while
+ * off screen; malformed files; no allocation per frame once playing; every
+ * MEM block back after BK_video_free + BK_image_exit. Exits 77 (skipped)
+ * when the build or the machine has no video backend that reads the clip.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#ifdef _WIN32
+# define WIN32_LEAN_AND_MEAN
+# include <windows.h>
+# define sleep_ms(ms) Sleep(ms)
+#else
+# include <time.h>
+# include <unistd.h>
+# define sleep_ms(ms) usleep((ms) * 1000)
+#endif
+
+#include "BUI_kit.h"
+#include "GPU_raster.h"
+#include "MEM_guardedalloc.h"
+
+static int g_fail = 0;
+#define CHECK(c) \
+ do { \
+ if (!(c)) { \
+ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); \
+ g_fail = 1; \
+ } \
+ } while (0)
+
+static double now_s(void)
+{
+#ifdef _WIN32
+ LARGE_INTEGER f, c;
+ QueryPerformanceFrequency(&f);
+ QueryPerformanceCounter(&c);
+ return (double)c.QuadPart / (double)f.QuadPart;
+#else
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec + ts.tv_nsec / 1e9;
+#endif
+}
+
+/* A host whose one timer the test loop fires. */
+static struct {
+ double due;
+ int timer, requests;
+} g_host;
+
+static void host_schedule(void *data, int ms, int timer)
+{
+ (void)data;
+ g_host.due = now_s() + ms / 1000.0;
+ g_host.timer = timer;
+}
+
+static void host_request(void *data)
+{
+ (void)data;
+ g_host.requests++;
+}
+
+#define W 200
+#define H 120
+
+/* Runs the context like a window loop for `seconds` (or until `until`
+ * returns nonzero); returns the frames rendered. */
+static int run(BKContext *ctx, BuiRaster *r, double seconds, int (*until)(void *), void *arg)
+{
+ const double end = now_s() + seconds;
+ int frames = 0;
+ while (now_s() < end && !(until && until(arg))) {
+ if (g_host.due > 0.0 && now_s() >= g_host.due) {
+ g_host.due = 0.0;
+ BK_timer(ctx, g_host.timer);
+ }
+ BK_image_poll();
+ if (BK_needs_render(ctx)) {
+ frames += BK_render(ctx, r, W, H, 0) > 0;
+ }
+ sleep_ms(1);
+ }
+ return frames;
+}
+
+static int loaded(void *v)
+{
+ return BK_video_state((BKVideo *)v) != BK_VIDEO_LOADING;
+}
+
+static int ended(void *v)
+{
+ return BK_video_state((BKVideo *)v) == BK_VIDEO_ENDED;
+}
+
+static uint64_t box_hash(const uint32_t *px)
+{
+ uint64_t h = 1469598103934665603ull;
+ for (int y = 10; y < 64; y++) {
+ for (int x = 10; x < 106; x++) {
+ h = (h ^ (px[y * W + x] & 0xFFFFFF)) * 1099511628211ull;
+ }
+ }
+ return h;
+}
+
+static int g_events[8];
+static void listener(BKVideo *v, int state, void *data)
+{
+ (void)v;
+ (void)data;
+ g_events[state & 7]++;
+}
+
+static void show(BKNode *canvas, BKVideo *v)
+{
+ static const float box[4] = {10, 10, 106, 64}; /* the clip's 96 x 54 */
+ BKCanvasItem it;
+ memset(&it, 0, sizeof(it));
+ it.id = 1;
+ it.type = BK_ITEM_IMAGE;
+ it.coords = box;
+ it.ncoords = 4;
+ it.video = v;
+ it.fit = BK_FIT_FILL;
+ BK_canvas_set_item(canvas, &it);
+}
+
+static void test_playback(const char *clip)
+{
+ BKNode *canvas = BK_node_new("canvas");
+ BKContext *ctx = BK_context_new(canvas);
+ BKHost host = {0};
+ host.request = host_request;
+ host.schedule = host_schedule;
+ BK_context_set_host(ctx, &host);
+ BuiRaster *r = BUI_raster_create(W, H);
+ const uint32_t *px = BUI_raster_pixels(r);
+
+ /* loads when first drawn, then plays (muted autoplay) */
+ BKVideo *v = BK_video_new(clip);
+ BK_video_set_listener(v, listener, NULL);
+ show(canvas, v);
+ const double t0 = now_s();
+ BK_render(ctx, r, W, H, 1);
+ run(ctx, r, 5.0, loaded, v);
+ printf("first frame after %.1f ms; backends: %s\n", (now_s() - t0) * 1000.0,
+ BK_video_backends());
+ int w = 0, h = 0;
+ CHECK(BK_video_state(v) == BK_VIDEO_PLAYING);
+ CHECK(BK_video_size(v, &w, &h) && w == 96 && h == 54);
+ CHECK(fabs(BK_video_duration(v) - 2.0) < 0.15);
+ CHECK(g_events[BK_VIDEO_PLAYING] == 1);
+
+ /* it plays: the picture moves and the time goes on */
+ uint64_t last = box_hash(px);
+ int changes = 0;
+ for (int i = 0; i < 10; i++) {
+ run(ctx, r, 0.06, NULL, NULL);
+ const uint64_t c = box_hash(px);
+ changes += c != last;
+ last = c;
+ }
+ const double t_play = BK_video_time(v);
+ printf("playing 0.6 s: %d frame changes, at %.2f s\n", changes, t_play);
+ CHECK(changes >= 3 && t_play > 0.2);
+
+ /* paused: nothing moves */
+ BK_video_pause(v);
+ run(ctx, r, 0.1, NULL, NULL);
+ const uint64_t held = box_hash(px);
+ const double t_held = BK_video_time(v);
+ run(ctx, r, 0.3, NULL, NULL);
+ CHECK(box_hash(px) == held && BK_video_time(v) == t_held);
+ CHECK(BK_video_state(v) == BK_VIDEO_PAUSED);
+
+ /* seek while paused: that frame shows */
+ BK_video_seek(v, 1.5);
+ run(ctx, r, 0.4, NULL, NULL);
+ printf("seek to 1.5 s: at %.3f s\n", BK_video_time(v));
+ CHECK(fabs(BK_video_time(v) - 1.5) < 0.11);
+ CHECK(box_hash(px) != held);
+
+ /* no allocation per frame once playing (MEM: requests and frames recycle) */
+ BK_video_seek(v, 0.0);
+ BK_video_play(v);
+ run(ctx, r, 0.3, NULL, NULL);
+ const size_t allocs = MEM_get_allocations_total();
+ const unsigned blocks = MEM_get_memory_blocks_in_use();
+ const double ta = BK_video_time(v);
+ run(ctx, r, 0.6, NULL, NULL);
+ const double frames_played = (BK_video_time(v) - ta) * 10.0;
+ printf("playing %.0f frames: %zu MEM allocations, %d blocks change\n", frames_played,
+ MEM_get_allocations_total() - allocs, (int)MEM_get_memory_blocks_in_use() - (int)blocks);
+ CHECK(MEM_get_allocations_total() == allocs);
+
+ /* rate and end: 4x from 1 s plays the rest in 0.25 s, then it ends (once) */
+ BK_video_seek(v, 1.0);
+ BK_video_set_rate(v, 4.0);
+ const double te = now_s();
+ run(ctx, r, 3.0, ended, v);
+ printf("4x from 1 s ended after %.2f s\n", now_s() - te);
+ CHECK(BK_video_state(v) == BK_VIDEO_ENDED && g_events[BK_VIDEO_ENDED] == 1);
+ CHECK(now_s() - te < 1.0);
+
+ /* loop: it goes round */
+ BK_video_set_loop(v, 1);
+ BK_video_play(v);
+ double lowest = 10.0, highest = 0.0;
+ for (int i = 0; i < 40; i++) {
+ run(ctx, r, 0.03, NULL, NULL);
+ const double t = BK_video_time(v);
+ lowest = t < lowest ? t : lowest;
+ highest = t > highest ? t : highest;
+ }
+ printf("looping 1.2 s at 4x: time between %.2f and %.2f\n", lowest, highest);
+ CHECK(BK_video_state(v) == BK_VIDEO_PLAYING && lowest < 0.5 && highest > 1.5);
+
+ /* off screen: no decoding, no timers, no renders */
+ BK_canvas_delete(canvas, -1);
+ BK_render(ctx, r, W, H, 1);
+ BKImageStats st;
+ for (int i = 0; i < 200; i++) {
+ run(ctx, r, 0.01, NULL, NULL);
+ BK_image_stats(&st);
+ if (st.pending == 0) {
+ break;
+ }
+ }
+ run(ctx, r, 0.05, NULL, NULL);
+ BK_image_stats(&st);
+ const int decoded = st.decoded, requests = g_host.requests;
+ const double tv = BK_video_time(v);
+ const int frames = run(ctx, r, 0.5, NULL, NULL);
+ BK_image_stats(&st);
+ printf("off screen for 0.5 s: %d decodes, %d renders, %d render requests\n",
+ st.decoded - decoded, frames, g_host.requests - requests);
+ CHECK(st.decoded == decoded && frames == 0 && g_host.requests == requests);
+ CHECK(BK_video_time(v) == tv);
+
+ /* shown again: it resumes */
+ show(canvas, v);
+ BK_render(ctx, r, W, H, 1);
+ run(ctx, r, 0.3, NULL, NULL);
+ CHECK(BK_video_time(v) != tv);
+
+ BK_canvas_delete(canvas, -1);
+ BK_video_free(v);
+ BK_context_free(ctx);
+ BK_node_free(canvas);
+ BUI_raster_free(r);
+}
+
+/* Truncated and bit-flipped copies: an error state or frames, never a crash. */
+static void test_malformed(const char *clip)
+{
+ FILE *f = fopen(clip, "rb");
+ if (!f) {
+ CHECK(0);
+ return;
+ }
+ static unsigned char data[1 << 16];
+ const size_t n = fread(data, 1, sizeof(data), f);
+ fclose(f);
+ BKNode *canvas = BK_node_new("canvas");
+ BKContext *ctx = BK_context_new(canvas);
+ BuiRaster *r = BUI_raster_create(W, H);
+ char path[600];
+ snprintf(path, sizeof(path), "%s/bad.mp4", BUI_TEST_TMP);
+ int errors = 0, played = 0;
+ srand(7);
+ for (int it = 0; it < 24; it++) {
+ static unsigned char bad[1 << 16];
+ memcpy(bad, data, n);
+ size_t len = n;
+ if (it % 3 == 0) {
+ len = (size_t)rand() % n; /* truncated */
+ }
+ else {
+ for (int k = 0; k < 1 + it; k++) { /* bytes flipped */
+ bad[(size_t)rand() % n] ^= (unsigned char)(1 + rand() % 255);
+ }
+ }
+ FILE *o = fopen(path, "wb");
+ fwrite(bad, 1, len, o);
+ fclose(o);
+ BKVideo *v = BK_video_new(path);
+ show(canvas, v);
+ BK_render(ctx, r, W, H, 1);
+ run(ctx, r, 2.0, loaded, v);
+ run(ctx, r, 0.2, NULL, NULL); /* a few frames of whatever decodes */
+ errors += BK_video_state(v) == BK_VIDEO_ERROR;
+ played += BK_video_state(v) == BK_VIDEO_PLAYING || BK_video_state(v) == BK_VIDEO_ENDED;
+ BK_canvas_delete(canvas, -1);
+ BK_render(ctx, r, W, H, 1);
+ BK_video_free(v);
+ }
+ remove(path);
+ printf("24 malformed clips: %d errors, %d played, no crash\n", errors, played);
+ CHECK(errors + played > 0);
+ BK_context_free(ctx);
+ BK_node_free(canvas);
+ BUI_raster_free(r);
+}
+
+int main(void)
+{
+ setvbuf(stdout, NULL, _IONBF, 0);
+ if (!BK_video_backends()[0]) {
+ printf("video_test: no video backend in this build, skipped\n");
+ return 77;
+ }
+ const unsigned blocks0 = MEM_get_memory_blocks_in_use();
+ const size_t bytes0 = MEM_get_memory_in_use();
+ char clip[600];
+ snprintf(clip, sizeof(clip), "%s/%s", BUI_TEST_VIDEOS, getenv("BUI_TEST_CLIP") ? getenv("BUI_TEST_CLIP") : "clip.mp4");
+ {
+ /* the machine may lack the decoder (Windows N without the Media Feature Pack) */
+ BKVideo *probe = BK_video_new(clip);
+ BKNode *canvas = BK_node_new("canvas");
+ BKContext *ctx = BK_context_new(canvas);
+ BuiRaster *r = BUI_raster_create(W, H);
+ show(canvas, probe);
+ BK_render(ctx, r, W, H, 1);
+ run(ctx, r, 5.0, loaded, probe);
+ const int state = BK_video_state(probe);
+ BK_canvas_delete(canvas, -1);
+ BK_video_free(probe);
+ BK_context_free(ctx);
+ BK_node_free(canvas);
+ BUI_raster_free(r);
+ BK_image_exit();
+ if (state == BK_VIDEO_ERROR || state == BK_VIDEO_LOADING) {
+ printf("video_test: %s does not decode %s here, skipped\n", BK_video_backends(), clip);
+ return 77;
+ }
+ }
+ memset(g_events, 0, sizeof(g_events));
+ test_playback(clip);
+ test_malformed(clip);
+ BK_image_exit();
+ printf("memory: %u blocks, %zu bytes before; %u blocks, %zu bytes after\n", blocks0, bytes0,
+ MEM_get_memory_blocks_in_use(), MEM_get_memory_in_use());
+ CHECK(MEM_get_memory_blocks_in_use() == blocks0);
+ CHECK(MEM_get_memory_in_use() == bytes0);
+ printf(g_fail ? "video_test: FAILED\n" : "video_test: ok\n");
+ return g_fail;
+}
diff --git a/tests/videos/clip.mp4 b/tests/videos/clip.mp4
new file mode 100644
index 0000000..f46b45b
Binary files /dev/null and b/tests/videos/clip.mp4 differ
diff --git a/tests/videos/clip_av1.mp4 b/tests/videos/clip_av1.mp4
new file mode 100644
index 0000000..81f9fea
Binary files /dev/null and b/tests/videos/clip_av1.mp4 differ