Skip to content

Visualize Your Ideas With Code

feat: three.js integration

aarthificial79cc975ecaa3
diff --git a/package-lock.json b/package-lock.json
index d90e738e..2f8e688f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -13,6 +13,7 @@
"image-size": "^1.0.1",
"konva": "^8.3.2",
"mix-color": "^1.1.2",
+ "three": "^0.138.3",
"ts-loader": "^9.2.6",
"typescript": "^4.5.5",
"url-loader": "^4.1.1",
@@ -25,6 +26,7 @@
"mc-serve": "tools/serve.mjs"
},
"devDependencies": {
+ "@types/three": "^0.138.0",
"node-loader": "^2.0.0",
"prettier": "^2.5.1",
"webpack-cli": "^4.9.2"
@@ -231,6 +233,12 @@
"@types/node": "*"
}
},
+ "node_modules/@types/three": {
+ "version": "0.138.0",
+ "resolved": "https://registry.npmjs.org/@types/three/-/three-0.138.0.tgz",
+ "integrity": "sha512-D8AoV7h2kbCfrv/DcebHOFh1WDwyus3HdooBkAwcBikXArdqnsQ38PQ85JCunnvun160oA9jz53GszF3zch3tg==",
+ "dev": true
+ },
"node_modules/@types/ws": {
"version": "8.2.2",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.2.2.tgz",
@@ -3623,6 +3631,11 @@
"node": ">= 8"
}
},
+ "node_modules/three": {
+ "version": "0.138.3",
+ "resolved": "https://registry.npmjs.org/three/-/three-0.138.3.tgz",
+ "integrity": "sha512-4t1cKC8gimNyJChJbaklg8W/qj3PpsLJUIFm5LIuAy/hVxxNm1ru2FGTSfbTSsuHmC/7ipsyuGKqrSAKLNtkzg=="
+ },
"node_modules/thunky": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz",
@@ -4381,6 +4394,12 @@
"@types/node": "*"
}
},
+ "@types/three": {
+ "version": "0.138.0",
+ "resolved": "https://registry.npmjs.org/@types/three/-/three-0.138.0.tgz",
+ "integrity": "sha512-D8AoV7h2kbCfrv/DcebHOFh1WDwyus3HdooBkAwcBikXArdqnsQ38PQ85JCunnvun160oA9jz53GszF3zch3tg==",
+ "dev": true
+ },
"@types/ws": {
"version": "8.2.2",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.2.2.tgz",
@@ -6870,6 +6889,11 @@
"terser": "^5.7.2"
}
},
+ "three": {
+ "version": "0.138.3",
+ "resolved": "https://registry.npmjs.org/three/-/three-0.138.3.tgz",
+ "integrity": "sha512-4t1cKC8gimNyJChJbaklg8W/qj3PpsLJUIFm5LIuAy/hVxxNm1ru2FGTSfbTSsuHmC/7ipsyuGKqrSAKLNtkzg=="
+ },
"thunky": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz",
diff --git a/package.json b/package.json
index c2ceef12..14ceb765 100644
--- a/package.json
+++ b/package.json
@@ -17,6 +17,7 @@
"image-size": "^1.0.1",
"konva": "^8.3.2",
"mix-color": "^1.1.2",
+ "three": "^0.138.3",
"ts-loader": "^9.2.6",
"typescript": "^4.5.5",
"url-loader": "^4.1.1",
@@ -34,6 +35,7 @@
"tsconfig.project.json"
],
"devDependencies": {
+ "@types/three": "^0.138.0",
"node-loader": "^2.0.0",
"prettier": "^2.5.1",
"webpack-cli": "^4.9.2"
diff --git a/src/components/ThreeView.ts b/src/components/ThreeView.ts
new file mode 100644
index 00000000..59eb4124
--- /dev/null
+++ b/src/components/ThreeView.ts
@@ -0,0 +1,181 @@
+import {LayoutShape, LayoutShapeConfig} from './LayoutShape';
+import {Size} from '../types';
+import {Util} from 'konva/lib/Util';
+import {Context} from 'konva/lib/Context';
+
+import * as THREE from 'three';
+import {GetSet} from 'konva/lib/types';
+import {Factory} from 'konva/lib/Factory';
+
+export interface ThreeViewConfig extends LayoutShapeConfig {
+ canvasSize: Size;
+ cameraScale?: number;
+ quality?: number;
+ skipFrames?: number;
+}
+
+interface Pool<T> {
+ borrow(): T;
+ dispose(object: T): void;
+}
+
+class CanvasPool implements Pool<HTMLCanvasElement> {
+ private pool: HTMLCanvasElement[] = [];
+
+ public borrow(): HTMLCanvasElement {
+ if (this.pool.length) {
+ return this.pool.pop();
+ } else {
+ return Util.createCanvasElement();
+ }
+ }
+
+ public dispose(canvas: HTMLCanvasElement) {
+ this.pool.push(canvas);
+ }
+}
+
+const canvasPool2D = new CanvasPool();
+const canvasPool3D = new CanvasPool();
+
+export class ThreeView extends LayoutShape {
+ public canvasSize: GetSet<Size, this>;
+ public cameraScale: GetSet<number, this>;
+ public quality: GetSet<number, this>;
+ public skipFrames: GetSet<number, this>;
+ public readonly scene: THREE.Scene;
+ public readonly camera: THREE.OrthographicCamera;
+
+ private readonly threeCanvas: HTMLCanvasElement;
+ private readonly copyCanvas: HTMLCanvasElement;
+ private readonly renderer: THREE.WebGLRenderer;
+ private readonly context: WebGLRenderingContext;
+ private readonly copyContext: CanvasRenderingContext2D;
+
+ private copyData: ImageData;
+ private pixels: Uint8ClampedArray;
+ private renderedFrames: number = 0;
+
+ public constructor(config?: ThreeViewConfig) {
+ super(config);
+ this.threeCanvas = canvasPool3D.borrow();
+ this.copyCanvas = canvasPool2D.borrow();
+ this.copyContext = this.copyCanvas.getContext('2d');
+
+ this.scene = new THREE.Scene();
+ this.scene.background = new THREE.Color(0x568585);
+ this.camera = new THREE.OrthographicCamera(-1, 1, 1, -1, 1, 1000);
+ this.renderer = new THREE.WebGLRenderer({
+ canvas: this.threeCanvas,
+ antialias: true,
+ });
+ this.context = this.renderer.getContext();
+ this.camera.position.z = 3;
+ this.camera.position.y = 2;
+
+ this.handleCanvasSizeChange();
+ }
+
+ destroy(): this {
+ this.renderer.dispose();
+ canvasPool2D.dispose(this.copyCanvas);
+ canvasPool3D.dispose(this.threeCanvas);
+
+ return super.destroy();
+ }
+
+ private handleCanvasSizeChange() {
+ if (!this.renderer) return;
+
+ const size = {...this.canvasSize()};
+
+ const ratio = size.width / size.height;
+ const scale = this.cameraScale() / 2;
+ this.camera.left = -ratio * scale;
+ this.camera.right = ratio * scale;
+ this.camera.bottom = -scale;
+ this.camera.top = scale;
+
+ size.width *= this.quality();
+ size.height *= this.quality();
+ this.renderer.setSize(size.width, size.height);
+ this.camera.updateProjectionMatrix();
+ this.copyCanvas.width = size.width;
+ this.copyCanvas.height = size.height;
+ this.copyData = this.copyContext.createImageData(size.width, size.height);
+ this.pixels = new Uint8ClampedArray(size.width * size.height * 4);
+ }
+
+ getLayoutSize(): Size {
+ return this.canvasSize();
+ }
+
+ _sceneFunc(context: Context) {
+ const scale = this.quality();
+ const size = {...this.canvasSize()};
+ size.width *= scale;
+ size.height *= scale;
+
+ if (this.renderedFrames < 1) {
+ this.renderedFrames = this.skipFrames();
+ this.renderer.render(this.scene, this.camera);
+ this.context.readPixels(
+ 0,
+ 0,
+ size.width,
+ size.height,
+ this.context.RGBA,
+ this.context.UNSIGNED_BYTE,
+ this.pixels,
+ );
+
+ this.copyData.data.set(this.pixels);
+ this.copyContext.putImageData(this.copyData, 0, 0);
+ } else {
+ this.renderedFrames--;
+ }
+
+ context.save();
+ context._context.imageSmoothingEnabled = false;
+ context.scale(1 / scale, 1 / -scale);
+ context.drawImage(
+ this.copyCanvas,
+ // this.copyImage,
+ 0,
+ 0,
+ size.width,
+ size.height,
+ size.width / -2,
+ size.height / -2,
+ size.width,
+ size.height,
+ );
+ context.restore();
+ }
+}
+
+Factory.addGetterSetter(
+ ThreeView,
+ 'canvasSize',
+ {width: 1, height: 1},
+ undefined,
+ //@ts-ignore
+ ThreeView.handleCanvasSizeChange,
+);
+Factory.addGetterSetter(
+ ThreeView,
+ 'cameraScale',
+ 1,
+ undefined,
+ //@ts-ignore
+ ThreeView.handleCanvasSizeChange,
+);
+Factory.addGetterSetter(
+ ThreeView,
+ 'quality',
+ 1,
+ undefined,
+ //@ts-ignore
+ ThreeView.handleCanvasSizeChange,
+);
+Factory.addGetterSetter(ThreeView, 'skipFrames', 0);
diff --git a/src/components/index.ts b/src/components/index.ts
index 2c73e7a0..fd29ed44 100644
--- a/src/components/index.ts
+++ b/src/components/index.ts
@@ -1,3 +1,8 @@
export * from './Arrow';
export * from './Connection';
-export * from './TextContent';
\ No newline at end of file
+export * from './Debug';
+export * from './Layout';
+export * from './Sprite';
+export * from './Surface';
+export * from './TextContent';
+export * from './ThreeView';
\ No newline at end of file
diff --git a/src/global.d.ts b/src/global.d.ts
index 03fb3f4b..06090f09 100644
--- a/src/global.d.ts
+++ b/src/global.d.ts
@@ -1,4 +1,9 @@
declare module "*.png" {
const value: any;
export = value;
+}
+
+declare module "*.glsl" {
+ const value: string;
+ export = value;
}
\ No newline at end of file
diff --git a/src/materials/MeshBoneMaterial.ts b/src/materials/MeshBoneMaterial.ts
new file mode 100644
index 00000000..f9c62f50
--- /dev/null
+++ b/src/materials/MeshBoneMaterial.ts
@@ -0,0 +1,86 @@
+import {
+ MaterialParameters,
+ ShaderLib,
+ ShaderMaterial,
+ Texture,
+ Vector4,
+} from 'three';
+
+import fragmentShader from 'MC/shaders/bone_highlight_fragment.glsl';
+import vertexShader from 'MC/shaders/bone_highlight_vertex.glsl';
+
+export interface MeshBoneMaterialParameters extends MaterialParameters {
+ highlight?: number;
+ map?: Texture;
+ wireframe?: boolean;
+ highlightColor?: Vector4;
+}
+
+export class MeshBoneMaterial extends ShaderMaterial {
+ public set map(value: Texture) {
+ if (value) {
+ this.uniforms.map = {value};
+ } else {
+ delete this.uniforms.map;
+ }
+ }
+
+ public get map(): Texture {
+ return this.uniforms.map?.value ?? null;
+ }
+
+ public set highlight(value: number | null) {
+ this.uniforms.highlight = {value: value ?? -1};
+ }
+
+ public get highlight(): number | null {
+ return this.uniforms.highlight?.value === -1
+ ? null
+ : this.uniforms.highlight?.value;
+ }
+
+ public set highlightColor(value: Vector4) {
+ this.uniforms.highlightDiffuse = {value};
+ }
+
+ public get highlightColor(): Vector4 {
+ return this.uniforms.highlightDiffuse?.value;
+ }
+
+ public set transparency(value: number) {
+ this.uniforms.opacity.value = value;
+ }
+
+ public get transparency(): number {
+ return this.uniforms.opacity.value;
+ }
+
+ public set wireframeOverlay(value: boolean) {
+ if (value) {
+ this.defines.USE_WIREFRAME = {value: true};
+ } else {
+ delete this.defines.USE_WIREFRAME;
+ }
+ }
+
+ public get wireframeOverlay(): boolean {
+ return !!this.defines.USE_WIREFRAME;
+ }
+
+ public constructor(parameters: MeshBoneMaterialParameters = {}) {
+ const {map, highlight, highlightColor, opacity, wireframe, ...rest} =
+ parameters;
+ super({
+ ...rest,
+ fragmentShader,
+ vertexShader,
+ });
+
+ this.uniforms = {...ShaderLib.basic.uniforms};
+ this.map = map;
+ this.highlight = highlight;
+ this.highlightColor = highlightColor ?? new Vector4(0, 0, 0, 0);
+ this.transparency = opacity;
+ this.wireframeOverlay = wireframe;
+ }
+}
diff --git a/src/shaders/bone_highlight_fragment.glsl b/src/shaders/bone_highlight_fragment.glsl
new file mode 100644
index 00000000..fd097cd5
--- /dev/null
+++ b/src/shaders/bone_highlight_fragment.glsl
@@ -0,0 +1,81 @@
+uniform vec3 diffuse;
+uniform float opacity;
+uniform vec4 highlightDiffuse;
+
+varying float vHighlight;
+varying vec3 vCenter;
+#ifndef FLAT_SHADED
+ varying vec3 vNormal;
+#endif
+
+
+#include <common>
+#include <dithering_pars_fragment>
+#include <color_pars_fragment>
+#include <uv_pars_fragment>
+#include <uv2_pars_fragment>
+#include <map_pars_fragment>
+#include <alphamap_pars_fragment>
+#include <alphatest_pars_fragment>
+#include <aomap_pars_fragment>
+#include <lightmap_pars_fragment>
+#include <envmap_common_pars_fragment>
+#include <envmap_pars_fragment>
+#include <cube_uv_reflection_fragment>
+#include <fog_pars_fragment>
+#include <specularmap_pars_fragment>
+#include <logdepthbuf_pars_fragment>
+#include <clipping_planes_pars_fragment>
+
+void main() {
+ #include <clipping_planes_fragment>
+ vec4 diffuseColor = vec4( diffuse, opacity );
+ #include <logdepthbuf_fragment>
+ #include <map_fragment>
+ #include <color_fragment>
+ #include <alphamap_fragment>
+ #include <alphatest_fragment>
+ #include <specularmap_fragment>
+ ReflectedLight reflectedLight = ReflectedLight( vec3( 0.0 ), vec3( 0.0 ), vec3( 0.0 ), vec3( 0.0 ) );
+ #ifdef USE_LIGHTMAP
+ vec4 lightMapTexel= texture2D( lightMap, vUv2 );
+ reflectedLight.indirectDiffuse += lightMapTexel.rgb * lightMapIntensity;
+ #else
+ reflectedLight.indirectDiffuse += vec3( 1.0 );
+ #endif
+ #include <aomap_fragment>
+ reflectedLight.indirectDiffuse *= diffuseColor.rgb;
+ vec3 outgoingLight = reflectedLight.indirectDiffuse;
+ #include <envmap_fragment>
+ #include <output_fragment>
+ #include <tonemapping_fragment>
+ #include <encodings_fragment>
+ #include <fog_fragment>
+ #include <premultiplied_alpha_fragment>
+ #include <dithering_fragment>
+
+ #ifdef USE_WIREFRAME
+ float thickness = 1.0;
+ vec3 afwidth = fwidth( vCenter.xyz );
+ vec3 edge3 = smoothstep( ( thickness - 1.0 ) * afwidth, thickness * afwidth, vCenter.xyz );
+ float edge = 1.0 - min( min( edge3.x, edge3.y ), edge3.z );
+
+ gl_FragColor = mix(
+ gl_FragColor,
+ mix(vec4(0.0, 0.0, 0.0, 1.0), highlightDiffuse, vHighlight),
+ edge
+ );
+ #else
+ gl_FragColor.rgb = mix(
+ gl_FragColor.rgb,
+ highlightDiffuse.rgb * mix((gl_FragColor.r + gl_FragColor.b + gl_FragColor.g) / 3.0, 1.0, 0.7),
+ vHighlight
+ );
+
+ gl_FragColor.a = mix(
+ mix(gl_FragColor.a, highlightDiffuse.a, vHighlight),
+ gl_FragColor.a,
+ step(gl_FragColor.a, 0.3)
+ );
+ #endif
+}
\ No newline at end of file
diff --git a/src/shaders/bone_highlight_vertex.glsl b/src/shaders/bone_highlight_vertex.glsl
new file mode 100644
index 00000000..71de35aa
--- /dev/null
+++ b/src/shaders/bone_highlight_vertex.glsl
@@ -0,0 +1,47 @@
+#include <common>
+#include <uv_pars_vertex>
+#include <uv2_pars_vertex>
+#include <envmap_pars_vertex>
+#include <color_pars_vertex>
+#include <fog_pars_vertex>
+#include <morphtarget_pars_vertex>
+#include <skinning_pars_vertex>
+#include <logdepthbuf_pars_vertex>
+#include <clipping_planes_pars_vertex>
+
+uniform float highlight;
+attribute vec3 center;
+
+varying float vHighlight;
+varying vec3 vCenter;
+
+void main() {
+ #include <uv_vertex>
+ #include <uv2_vertex>
+ #include <color_vertex>
+ #include <morphcolor_vertex>
+ #if defined ( USE_ENVMAP ) || defined ( USE_SKINNING )
+ #include <beginnormal_vertex>
+ #include <morphnormal_vertex>
+ #include <skinbase_vertex>
+ #include <skinnormal_vertex>
+ #include <defaultnormal_vertex>
+ #endif
+ #include <begin_vertex>
+ #include <morphtarget_vertex>
+ #include <skinning_vertex>
+ #include <project_vertex>
+ #include <logdepthbuf_vertex>
+ #include <clipping_planes_vertex>
+ #include <worldpos_vertex>
+ #include <envmap_vertex>
+ #include <fog_vertex>
+
+ #ifdef USE_SKINNING
+ vec4 highlights = skinWeight * step(skinIndex, vec4(highlight)) * step(vec4(highlight), skinIndex);
+ vHighlight = highlights.x + highlights.y + highlights.z + highlights.w;
+ #else
+ vHighlight = 0.0;
+ #endif
+ vCenter = center;
+}
\ No newline at end of file
diff --git a/tools/serve.mjs b/tools/serve.mjs
index f929d14c..7cefcb23 100644
--- a/tools/serve.mjs
+++ b/tools/serve.mjs
@@ -34,6 +34,10 @@ const compiler = webpack({
},
],
},
+ {
+ test: /\.glsl$/i,
+ type: 'asset/source',
+ },
],
},
resolve: {