Skip to content

Visualize Your Ideas With Code

feature: video component

aarthificiale937999f3d22
diff --git a/src/components/Video.tsx b/src/components/Video.tsx
new file mode 100644
index 00000000..40153307
--- /dev/null
+++ b/src/components/Video.tsx
@@ -0,0 +1,89 @@
+import {Shape, ShapeConfig} from 'konva/lib/Shape';
+import {getset, threadable} from '../decorators';
+import {Context} from 'konva/lib/Context';
+import {GetSet} from 'konva/lib/types';
+import {cancel, ThreadGenerator} from '../threading';
+import {waitFor} from '../animations';
+import {CanvasHelper, GeneratorHelper} from '../helpers';
+
+interface VideoConfig extends ShapeConfig {
+ frames: ImageBitmap[];
+ frame?: number;
+ fps?: number;
+ playing?: number;
+ radius?: number;
+}
+
+export class Video extends Shape {
+ @getset([])
+ public frames: GetSet<VideoConfig['frames'], this>;
+ @getset(0)
+ public frame: GetSet<VideoConfig['frame'], this>;
+ @getset(30)
+ public fps: GetSet<VideoConfig['fps'], this>;
+ @getset(true)
+ public playing: GetSet<VideoConfig['playing'], this>;
+ @getset(8)
+ public radius: GetSet<VideoConfig['radius'], this>;
+
+ private task: ThreadGenerator | null = null;
+
+ public constructor(config?: VideoConfig) {
+ super(config);
+ }
+
+ _sceneFunc(context: Context) {
+ const frames = this.frames();
+ context._context.clip(
+ CanvasHelper.roundRectPath(
+ new Path2D(),
+ 0,
+ 0,
+ this.width(),
+ this.height(),
+ this.radius()
+ )
+ );
+ if (frames.length) {
+ context.drawImage(frames[this.frame() % frames.length], 0, 0);
+ } else {
+ context._context.fillStyle = '#666666';
+ context.fillRect(0, 0, this.width(), this.height());
+ }
+ }
+
+ public play(): ThreadGenerator {
+ const runTask = this.playRunner();
+ if (this.task) {
+ const previousTask = this.task;
+ this.task = (function* (): ThreadGenerator {
+ yield* cancel(previousTask);
+ yield* runTask;
+ })();
+ GeneratorHelper.makeThreadable(this.task, runTask);
+ } else {
+ this.task = runTask;
+ }
+
+ return this.task;
+ }
+
+ @threadable()
+ public *stop() {
+ if (this.task) {
+ yield* cancel(this.task);
+ this.task = null;
+ }
+ }
+
+ @threadable('videoRunner')
+ private *playRunner(): ThreadGenerator {
+ this.frame(0);
+ while (this.task !== null) {
+ if (this.playing()) {
+ this.frame(this.frame() + 1);
+ }
+ yield* waitFor(1 / this.fps());
+ }
+ }
+}
diff --git a/src/global.d.ts b/src/global.d.ts
index 94d4cfe9..384a2033 100644
--- a/src/global.d.ts
+++ b/src/global.d.ts
@@ -28,6 +28,11 @@ declare module '*.wav?meta' {
export = value;
}
+declare module '*.mp4' {
+ const value: string;
+ export = value;
+}
+
declare namespace JSX {
type ElementClass = import('konva/lib/Node').Node;
interface ElementChildrenAttribute {
diff --git a/src/videos/MP4Demuxer.js b/src/videos/MP4Demuxer.js
new file mode 100644
index 00000000..5e24743e
--- /dev/null
+++ b/src/videos/MP4Demuxer.js
@@ -0,0 +1,175 @@
+import {createFile} from 'mp4box/dist/mp4box.all';
+
+class MP4Source {
+ constructor(uri) {
+ this.file = createFile();
+ this.file.onError = console.error.bind(console);
+ this.file.onReady = this.onReady.bind(this);
+ this.file.onSamples = this.onSamples.bind(this);
+
+ fetch(uri).then(response => {
+ const reader = response.body.getReader();
+ let offset = 0;
+ let mp4File = this.file;
+
+ function appendBuffers({done, value}) {
+ if(done) {
+ mp4File.flush();
+ return;
+ }
+
+ let buf = value.buffer;
+ buf.fileStart = offset;
+
+ offset += buf.byteLength;
+
+ mp4File.appendBuffer(buf);
+
+ return reader.read().then(appendBuffers);
+ }
+
+ return reader.read().then(appendBuffers);
+ })
+
+ this.info = null;
+ this._info_resolver = null;
+ }
+
+ onReady(info) {
+ // TODO: Generate configuration changes.
+ this.info = info;
+
+ if (this._info_resolver) {
+ this._info_resolver(info);
+ this._info_resolver = null;
+ }
+ }
+
+ getInfo() {
+ if (this.info)
+ return Promise.resolve(this.info);
+
+ return new Promise((resolver) => { this._info_resolver = resolver; });
+ }
+
+ getAvccBox() {
+ // TODO: make sure this is coming from the right track.
+ return this.file.moov.traks[0].mdia.minf.stbl.stsd.entries[0].avcC
+ }
+
+ start(track, onChunk) {
+ this._onChunk = onChunk;
+ this.file.setExtractionOptions(track.id);
+ this.file.start();
+ }
+
+ onSamples(track_id, ref, samples) {
+ for (const sample of samples) {
+ const type = sample.is_sync ? "key" : "delta";
+
+ const chunk = new EncodedVideoChunk({
+ type: type,
+ timestamp: sample.cts,
+ duration: sample.duration,
+ data: sample.data
+ });
+
+ this._onChunk(chunk);
+ }
+ }
+}
+
+class Writer {
+ constructor(size) {
+ this.data = new Uint8Array(size);
+ this.idx = 0;
+ this.size = size;
+ }
+
+ getData() {
+ if(this.idx != this.size)
+ throw "Mismatch between size reserved and sized used"
+
+ return this.data.slice(0, this.idx);
+ }
+
+ writeUint8(value) {
+ this.data.set([value], this.idx);
+ this.idx++;
+ }
+
+ writeUint16(value) {
+ // TODO: find a more elegant solution to endianess.
+ var arr = new Uint16Array(1);
+ arr[0] = value;
+ var buffer = new Uint8Array(arr.buffer);
+ this.data.set([buffer[1], buffer[0]], this.idx);
+ this.idx +=2;
+ }
+
+ writeUint8Array(value) {
+ this.data.set(value, this.idx);
+ this.idx += value.length;
+ }
+}
+
+export class MP4Demuxer {
+ constructor(uri) {
+ this.source = new MP4Source(uri);
+ }
+
+ getExtradata(avccBox) {
+ var i;
+ var size = 7;
+ for (i = 0; i < avccBox.SPS.length; i++) {
+ // nalu length is encoded as a uint16.
+ size+= 2 + avccBox.SPS[i].length;
+ }
+ for (i = 0; i < avccBox.PPS.length; i++) {
+ // nalu length is encoded as a uint16.
+ size+= 2 + avccBox.PPS[i].length;
+ }
+
+ var writer = new Writer(size);
+
+ writer.writeUint8(avccBox.configurationVersion);
+ writer.writeUint8(avccBox.AVCProfileIndication);
+ writer.writeUint8(avccBox.profile_compatibility);
+ writer.writeUint8(avccBox.AVCLevelIndication);
+ writer.writeUint8(avccBox.lengthSizeMinusOne + (63<<2));
+
+ writer.writeUint8(avccBox.nb_SPS_nalus + (7<<5));
+ for (i = 0; i < avccBox.SPS.length; i++) {
+ writer.writeUint16(avccBox.SPS[i].length);
+ writer.writeUint8Array(avccBox.SPS[i].nalu);
+ }
+
+ writer.writeUint8(avccBox.nb_PPS_nalus);
+ for (i = 0; i < avccBox.PPS.length; i++) {
+ writer.writeUint16(avccBox.PPS[i].length);
+ writer.writeUint8Array(avccBox.PPS[i].nalu);
+ }
+
+ return writer.getData();
+ }
+
+ async getConfig() {
+ let info = await this.source.getInfo();
+ this.track = info.videoTracks[0];
+
+ var extradata = this.getExtradata(this.source.getAvccBox());
+
+ let config = {
+ codec: this.track.codec,
+ codedHeight: this.track.video.height,
+ codedWidth: this.track.video.width,
+ description: extradata,
+ }
+
+ return Promise.resolve(config);
+ }
+
+ start(onChunk) {
+ this.source.start(this.track, onChunk);
+ }
+}
\ No newline at end of file
diff --git a/src/videos/loadVideo.ts b/src/videos/loadVideo.ts
new file mode 100644
index 00000000..5e7aa5de
--- /dev/null
+++ b/src/videos/loadVideo.ts
@@ -0,0 +1,58 @@
+import {MP4Demuxer} from './MP4Demuxer';
+import {useScene} from '../utils';
+
+const videos = new WeakMap<object, ImageBitmap[]>();
+const keys: Record<string, object> = {};
+
+export async function loadVideo(src: string) {
+ const key = getKey(src);
+ const previous = videos.get(key);
+ if (previous) {
+ return previous;
+ }
+
+ // TODO Replace with a more complex object to notify about changes etc.
+ const frames: ImageBitmap[] = [];
+ videos.set(key, frames);
+
+ // TODO Actually check if the project is being rendered.
+ const isRendering = false;
+ if (isRendering) {
+ await decode(src, frames);
+ } else {
+ decode(src, frames).catch(console.error);
+ }
+
+ return frames;
+}
+
+function getKey(src: string) {
+ keys[src] ??= {src};
+ return keys[src];
+}
+
+async function decode(src: string, frames: ImageBitmap[]) {
+ let error: DOMException = null;
+ const demuxer = new MP4Demuxer(src);
+ const decoder = new VideoDecoder({
+ output: frame => {
+ createImageBitmap(frame).then(value => {
+ frames.push(value);
+ frame.close();
+ });
+ },
+ error: e => {
+ error = e;
+ },
+ });
+
+ const config = await demuxer.getConfig();
+ decoder.configure(config);
+ demuxer.start((chunk: EncodedVideoChunk) => {
+ decoder.decode(chunk);
+ });
+ await decoder.flush();
+ if (error) {
+ throw error;
+ }
+}
diff --git a/tools/serve.mjs b/tools/serve.mjs
index 49ae995c..dd88b353 100644
--- a/tools/serve.mjs
+++ b/tools/serve.mjs
@@ -68,6 +68,10 @@ const compiler = webpack({
test: /\.glsl$/i,
type: 'asset/source',
},
+ {
+ test: /\.mp4/i,
+ type: 'asset',
+ },
{
test: /\.label$/i,
use: [
diff --git a/tsconfig.json b/tsconfig.json
index d75377db..f2e12ad3 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -7,6 +7,7 @@
"noImplicitAny": true,
"module": "esnext",
"target": "es2020",
+ "allowJs": true,
"moduleResolution": "node",
"resolveJsonModule": true,
"declaration": true,
diff --git a/tsconfig.project.json b/tsconfig.project.json
index 29c1086b..d033d561 100644
--- a/tsconfig.project.json
+++ b/tsconfig.project.json
@@ -4,6 +4,7 @@
"noImplicitAny": true,
"module": "esnext",
"target": "es2020",
+ "allowJs": true,
"moduleResolution": "node",
"resolveJsonModule": true,
"allowSyntheticDefaultImports": true,