Visualize Your Ideas With Code
feature: video component
aarthificiale937999f3d22
diff --git a/src/components/Video.tsx b/src/components/Video.tsxnew file mode 100644index 00000000..40153307--- /dev/null+++ b/src/components/Video.tsx@@ -0,0 +1,89 @@+import {Shape, ShapeConfig} from 'konva/lib/Shape';+import {getset, threadable} from '../decorators';+import {Context} from 'konva/lib/Context';+import {GetSet} from 'konva/lib/types';+import {cancel, ThreadGenerator} from '../threading';+import {waitFor} from '../animations';+import {CanvasHelper, GeneratorHelper} from '../helpers';++interface VideoConfig extends ShapeConfig {+ frames: ImageBitmap[];+ frame?: number;+ fps?: number;+ playing?: number;+ radius?: number;+}++export class Video extends Shape {+ @getset([])+ public frames: GetSet<VideoConfig['frames'], this>;+ @getset(0)+ public frame: GetSet<VideoConfig['frame'], this>;+ @getset(30)+ public fps: GetSet<VideoConfig['fps'], this>;+ @getset(true)+ public playing: GetSet<VideoConfig['playing'], this>;+ @getset(8)+ public radius: GetSet<VideoConfig['radius'], this>;++ private task: ThreadGenerator | null = null;++ public constructor(config?: VideoConfig) {+ super(config);+ }++ _sceneFunc(context: Context) {+ const frames = this.frames();+ context._context.clip(+ CanvasHelper.roundRectPath(+ new Path2D(),+ 0,+ 0,+ this.width(),+ this.height(),+ this.radius()+ )+ );+ if (frames.length) {+ context.drawImage(frames[this.frame() % frames.length], 0, 0);+ } else {+ context._context.fillStyle = '#666666';+ context.fillRect(0, 0, this.width(), this.height());+ }+ }++ public play(): ThreadGenerator {+ const runTask = this.playRunner();+ if (this.task) {+ const previousTask = this.task;+ this.task = (function* (): ThreadGenerator {+ yield* cancel(previousTask);+ yield* runTask;+ })();+ GeneratorHelper.makeThreadable(this.task, runTask);+ } else {+ this.task = runTask;+ }++ return this.task;+ }++ @threadable()+ public *stop() {+ if (this.task) {+ yield* cancel(this.task);+ this.task = null;+ }+ }++ @threadable('videoRunner')+ private *playRunner(): ThreadGenerator {+ this.frame(0);+ while (this.task !== null) {+ if (this.playing()) {+ this.frame(this.frame() + 1);+ }+ yield* waitFor(1 / this.fps());+ }+ }+}diff --git a/src/global.d.ts b/src/global.d.tsindex 94d4cfe9..384a2033 100644--- a/src/global.d.ts+++ b/src/global.d.ts@@ -28,6 +28,11 @@ declare module '*.wav?meta' {export = value;}+declare module '*.mp4' {+ const value: string;+ export = value;+}+declare namespace JSX {type ElementClass = import('konva/lib/Node').Node;interface ElementChildrenAttribute {diff --git a/src/videos/MP4Demuxer.js b/src/videos/MP4Demuxer.jsnew file mode 100644index 00000000..5e24743e--- /dev/null+++ b/src/videos/MP4Demuxer.js@@ -0,0 +1,175 @@+import {createFile} from 'mp4box/dist/mp4box.all';++class MP4Source {+ constructor(uri) {+ this.file = createFile();+ this.file.onError = console.error.bind(console);+ this.file.onReady = this.onReady.bind(this);+ this.file.onSamples = this.onSamples.bind(this);++ fetch(uri).then(response => {+ const reader = response.body.getReader();+ let offset = 0;+ let mp4File = this.file;++ function appendBuffers({done, value}) {+ if(done) {+ mp4File.flush();+ return;+ }++ let buf = value.buffer;+ buf.fileStart = offset;++ offset += buf.byteLength;++ mp4File.appendBuffer(buf);++ return reader.read().then(appendBuffers);+ }++ return reader.read().then(appendBuffers);+ })++ this.info = null;+ this._info_resolver = null;+ }++ onReady(info) {+ // TODO: Generate configuration changes.+ this.info = info;++ if (this._info_resolver) {+ this._info_resolver(info);+ this._info_resolver = null;+ }+ }++ getInfo() {+ if (this.info)+ return Promise.resolve(this.info);++ return new Promise((resolver) => { this._info_resolver = resolver; });+ }++ getAvccBox() {+ // TODO: make sure this is coming from the right track.+ return this.file.moov.traks[0].mdia.minf.stbl.stsd.entries[0].avcC+ }++ start(track, onChunk) {+ this._onChunk = onChunk;+ this.file.setExtractionOptions(track.id);+ this.file.start();+ }++ onSamples(track_id, ref, samples) {+ for (const sample of samples) {+ const type = sample.is_sync ? "key" : "delta";++ const chunk = new EncodedVideoChunk({+ type: type,+ timestamp: sample.cts,+ duration: sample.duration,+ data: sample.data+ });++ this._onChunk(chunk);+ }+ }+}++class Writer {+ constructor(size) {+ this.data = new Uint8Array(size);+ this.idx = 0;+ this.size = size;+ }++ getData() {+ if(this.idx != this.size)+ throw "Mismatch between size reserved and sized used"++ return this.data.slice(0, this.idx);+ }++ writeUint8(value) {+ this.data.set([value], this.idx);+ this.idx++;+ }++ writeUint16(value) {+ // TODO: find a more elegant solution to endianess.+ var arr = new Uint16Array(1);+ arr[0] = value;+ var buffer = new Uint8Array(arr.buffer);+ this.data.set([buffer[1], buffer[0]], this.idx);+ this.idx +=2;+ }++ writeUint8Array(value) {+ this.data.set(value, this.idx);+ this.idx += value.length;+ }+}++export class MP4Demuxer {+ constructor(uri) {+ this.source = new MP4Source(uri);+ }++ getExtradata(avccBox) {+ var i;+ var size = 7;+ for (i = 0; i < avccBox.SPS.length; i++) {+ // nalu length is encoded as a uint16.+ size+= 2 + avccBox.SPS[i].length;+ }+ for (i = 0; i < avccBox.PPS.length; i++) {+ // nalu length is encoded as a uint16.+ size+= 2 + avccBox.PPS[i].length;+ }++ var writer = new Writer(size);++ writer.writeUint8(avccBox.configurationVersion);+ writer.writeUint8(avccBox.AVCProfileIndication);+ writer.writeUint8(avccBox.profile_compatibility);+ writer.writeUint8(avccBox.AVCLevelIndication);+ writer.writeUint8(avccBox.lengthSizeMinusOne + (63<<2));++ writer.writeUint8(avccBox.nb_SPS_nalus + (7<<5));+ for (i = 0; i < avccBox.SPS.length; i++) {+ writer.writeUint16(avccBox.SPS[i].length);+ writer.writeUint8Array(avccBox.SPS[i].nalu);+ }++ writer.writeUint8(avccBox.nb_PPS_nalus);+ for (i = 0; i < avccBox.PPS.length; i++) {+ writer.writeUint16(avccBox.PPS[i].length);+ writer.writeUint8Array(avccBox.PPS[i].nalu);+ }++ return writer.getData();+ }++ async getConfig() {+ let info = await this.source.getInfo();+ this.track = info.videoTracks[0];++ var extradata = this.getExtradata(this.source.getAvccBox());++ let config = {+ codec: this.track.codec,+ codedHeight: this.track.video.height,+ codedWidth: this.track.video.width,+ description: extradata,+ }++ return Promise.resolve(config);+ }++ start(onChunk) {+ this.source.start(this.track, onChunk);+ }+}\ No newline at end of filediff --git a/src/videos/loadVideo.ts b/src/videos/loadVideo.tsnew file mode 100644index 00000000..5e7aa5de--- /dev/null+++ b/src/videos/loadVideo.ts@@ -0,0 +1,58 @@+import {MP4Demuxer} from './MP4Demuxer';+import {useScene} from '../utils';++const videos = new WeakMap<object, ImageBitmap[]>();+const keys: Record<string, object> = {};++export async function loadVideo(src: string) {+ const key = getKey(src);+ const previous = videos.get(key);+ if (previous) {+ return previous;+ }++ // TODO Replace with a more complex object to notify about changes etc.+ const frames: ImageBitmap[] = [];+ videos.set(key, frames);++ // TODO Actually check if the project is being rendered.+ const isRendering = false;+ if (isRendering) {+ await decode(src, frames);+ } else {+ decode(src, frames).catch(console.error);+ }++ return frames;+}++function getKey(src: string) {+ keys[src] ??= {src};+ return keys[src];+}++async function decode(src: string, frames: ImageBitmap[]) {+ let error: DOMException = null;+ const demuxer = new MP4Demuxer(src);+ const decoder = new VideoDecoder({+ output: frame => {+ createImageBitmap(frame).then(value => {+ frames.push(value);+ frame.close();+ });+ },+ error: e => {+ error = e;+ },+ });++ const config = await demuxer.getConfig();+ decoder.configure(config);+ demuxer.start((chunk: EncodedVideoChunk) => {+ decoder.decode(chunk);+ });+ await decoder.flush();+ if (error) {+ throw error;+ }+}diff --git a/tools/serve.mjs b/tools/serve.mjsindex 49ae995c..dd88b353 100644--- a/tools/serve.mjs+++ b/tools/serve.mjs@@ -68,6 +68,10 @@ const compiler = webpack({test: /\.glsl$/i,type: 'asset/source',},+ {+ test: /\.mp4/i,+ type: 'asset',+ },{test: /\.label$/i,use: [diff --git a/tsconfig.json b/tsconfig.jsonindex d75377db..f2e12ad3 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -7,6 +7,7 @@"noImplicitAny": true,"module": "esnext","target": "es2020",+ "allowJs": true,"moduleResolution": "node","resolveJsonModule": true,"declaration": true,diff --git a/tsconfig.project.json b/tsconfig.project.jsonindex 29c1086b..d033d561 100644--- a/tsconfig.project.json+++ b/tsconfig.project.json@@ -4,6 +4,7 @@"noImplicitAny": true,"module": "esnext","target": "es2020",+ "allowJs": true,"moduleResolution": "node","resolveJsonModule": true,"allowSyntheticDefaultImports": true,