Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
9.0 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {
	WEB_CAMERA_EFFECT_MASK_BAND_EPSILON,
	WEB_CAMERA_EFFECT_MASK_EDGE_SOFTNESS,
	WEB_CAMERA_EFFECT_MASK_GUIDE_RANGE_FALLOFF,
	WEB_CAMERA_EFFECT_MASK_GUIDE_SPATIAL_FALLOFF,
} from '@app/features/voice/utils/camera-effects/WebCameraEffectMask';
import {SEG_INPUT_EDGE} from '@app/features/voice/utils/camera-effects/WebSelfieSegmenter';

const VERTEX_SHADER_SOURCE = `#version 300 es
precision highp float;

out vec2 textureCoordinates;

void main() {
	vec2 positions[3] = vec2[3](
		vec2(-1.0, -1.0),
		vec2(3.0, -1.0),
		vec2(-1.0, 3.0)
	);
	vec2 position = positions[gl_VertexID];
	gl_Position = vec4(position, 0.0, 1.0);
	textureCoordinates = vec2((position.x + 1.0) * 0.5, (1.0 - position.y) * 0.5);
}
`;

const FRAGMENT_SHADER_SOURCE = `#version 300 es
precision highp float;

uniform sampler2D sourceTexture;
uniform sampler2D maskTexture;
in vec2 textureCoordinates;
out vec4 outputColour;

float luminance(vec3 colour) {
	return dot(colour, vec3(0.2126, 0.7152, 0.0722));
}

float refinedMask(vec2 coordinates, float guide) {
	vec2 maskSize = vec2(${SEG_INPUT_EDGE}.0);
	vec2 maskPosition = coordinates * maskSize - vec2(0.5);
	vec2 base = round(maskPosition);
	float maskTotal = 0.0;
	float weightTotal = 0.0;
	for (int y = -1; y <= 1; y += 1) {
		for (int x = -1; x <= 1; x += 1) {
			vec2 offset = vec2(float(x), float(y));
			vec2 sampleCoordinates = clamp((base + offset + vec2(0.5)) / maskSize, vec2(0.0), vec2(1.0));
			float sampleGuide = luminance(textureLod(sourceTexture, sampleCoordinates, 0.0).rgb);
			vec2 sampleDistance = maskPosition - (base + offset);
			float spatialWeight = exp(-float(${WEB_CAMERA_EFFECT_MASK_GUIDE_SPATIAL_FALLOFF}) * dot(sampleDistance, sampleDistance));
			float rangeWeight = exp(-float(${WEB_CAMERA_EFFECT_MASK_GUIDE_RANGE_FALLOFF}) * abs(guide - sampleGuide));
			float weight = spatialWeight * rangeWeight;
			maskTotal += textureLod(maskTexture, sampleCoordinates, 0.0).a * weight;
			weightTotal += weight;
		}
	}
	return maskTotal / max(weightTotal, 0.0001);
}

void main() {
	float guide = luminance(texture(sourceTexture, textureCoordinates).rgb);
	float coarse = textureLod(maskTexture, textureCoordinates, 0.0).a;
	float alpha = step(0.5, coarse);
	if (coarse > ${WEB_CAMERA_EFFECT_MASK_BAND_EPSILON} && coarse < 1.0 - ${WEB_CAMERA_EFFECT_MASK_BAND_EPSILON}) {
		float refined = refinedMask(textureCoordinates, guide);
		float curve = clamp((refined - 0.5) / (2.0 * ${WEB_CAMERA_EFFECT_MASK_EDGE_SOFTNESS}) + 0.5, 0.0, 1.0);
		alpha = curve * curve * (3.0 - 2.0 * curve);
	}
	outputColour = vec4(0.0, 0.0, 0.0, alpha);
}
`;

function createShader(gl: WebGL2RenderingContext, type: number, source: string): WebGLShader {
	const shader = gl.createShader(type);
	if (shader == null) {
		throw new Error('Camera mask refinement could not allocate a WebGL shader');
	}
	gl.shaderSource(shader, source);
	gl.compileShader(shader);
	if (gl.getShaderParameter(shader, gl.COMPILE_STATUS) === true) {
		return shader;
	}
	const diagnostic = gl.getShaderInfoLog(shader) ?? 'no shader diagnostic was provided';
	gl.deleteShader(shader);
	throw new Error(`Camera mask refinement WebGL shader compilation failed: ${diagnostic}`);
}

function createProgram(gl: WebGL2RenderingContext): WebGLProgram {
	const vertexShader = createShader(gl, gl.VERTEX_SHADER, VERTEX_SHADER_SOURCE);
	let fragmentShader: WebGLShader | null = null;
	let program: WebGLProgram | null = null;
	try {
		fragmentShader = createShader(gl, gl.FRAGMENT_SHADER, FRAGMENT_SHADER_SOURCE);
		program = gl.createProgram();
		if (program == null) {
			throw new Error('Camera mask refinement could not allocate a WebGL program');
		}
		gl.attachShader(program, vertexShader);
		gl.attachShader(program, fragmentShader);
		gl.linkProgram(program);
		if (gl.getProgramParameter(program, gl.LINK_STATUS) !== true) {
			const diagnostic = gl.getProgramInfoLog(program) ?? 'no program diagnostic was provided';
			throw new Error(`Camera mask refinement WebGL program linking failed: ${diagnostic}`);
		}
		return program;
	} catch (error) {
		if (program != null) {
			gl.deleteProgram(program);
		}
		throw error;
	} finally {
		gl.deleteShader(vertexShader);
		if (fragmentShader != null) {
			gl.deleteShader(fragmentShader);
		}
	}
}

function requireUniform(gl: WebGL2RenderingContext, program: WebGLProgram, name: string): WebGLUniformLocation {
	const location = gl.getUniformLocation(program, name);
	if (location == null) {
		throw new Error(`Camera mask refinement WebGL uniform is unavailable: ${name}`);
	}
	return location;
}

function createTexture(gl: WebGL2RenderingContext, unit: number): WebGLTexture {
	const texture = gl.createTexture();
	if (texture == null) {
		throw new Error('Camera mask refinement could not allocate a WebGL texture');
	}
	gl.activeTexture(gl.TEXTURE0 + unit);
	gl.bindTexture(gl.TEXTURE_2D, texture);
	gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
	gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
	gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
	gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
	return texture;
}

export class WebCameraEffectWebGLMaskRefiner {
	private width = 0;
	private height = 0;
	private uploadedMaskRevision = -1;
	private disposed = false;

	private constructor(
		private readonly canvas: OffscreenCanvas,
		private readonly gl: WebGL2RenderingContext,
		private readonly program: WebGLProgram,
		private readonly vertexArray: WebGLVertexArrayObject,
		private readonly sourceTexture: WebGLTexture,
		private readonly maskTexture: WebGLTexture,
	) {}

	static create(): WebCameraEffectWebGLMaskRefiner | null {
		const canvas = new OffscreenCanvas(1, 1);
		const gl = canvas.getContext('webgl2', {
			alpha: true,
			antialias: false,
			depth: false,
			premultipliedAlpha: false,
			preserveDrawingBuffer: true,
			stencil: false,
		});
		if (gl == null) {
			return null;
		}
		let program: WebGLProgram | null = null;
		let vertexArray: WebGLVertexArrayObject | null = null;
		let sourceTexture: WebGLTexture | null = null;
		let maskTexture: WebGLTexture | null = null;
		try {
			program = createProgram(gl);
			vertexArray = gl.createVertexArray();
			if (vertexArray == null) {
				throw new Error('Camera mask refinement could not allocate a WebGL vertex array');
			}
			sourceTexture = createTexture(gl, 0);
			maskTexture = createTexture(gl, 1);
			gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, SEG_INPUT_EDGE, SEG_INPUT_EDGE, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
			gl.useProgram(program);
			gl.uniform1i(requireUniform(gl, program, 'sourceTexture'), 0);
			gl.uniform1i(requireUniform(gl, program, 'maskTexture'), 1);
			gl.bindVertexArray(vertexArray);
			gl.disable(gl.BLEND);
			gl.disable(gl.CULL_FACE);
			gl.disable(gl.DEPTH_TEST);
			gl.disable(gl.DITHER);
			return new WebCameraEffectWebGLMaskRefiner(canvas, gl, program, vertexArray, sourceTexture, maskTexture);
		} catch (error) {
			if (sourceTexture != null) gl.deleteTexture(sourceTexture);
			if (maskTexture != null) gl.deleteTexture(maskTexture);
			if (vertexArray != null) gl.deleteVertexArray(vertexArray);
			if (program != null) gl.deleteProgram(program);
			gl.getExtension('WEBGL_lose_context')?.loseContext();
			throw error;
		}
	}

	refine(
		source: VideoFrame,
		mask: OffscreenCanvas,
		maskRevision: number,
		width: number,
		height: number,
	): OffscreenCanvas | null {
		if (this.disposed) {
			throw new Error('Cannot refine a camera mask with a disposed WebGL owner');
		}
		if (this.gl.isContextLost()) {
			return null;
		}
		if (this.width !== width || this.height !== height) {
			this.width = width;
			this.height = height;
			this.canvas.width = width;
			this.canvas.height = height;
			this.gl.viewport(0, 0, width, height);
			this.gl.activeTexture(this.gl.TEXTURE0);
			this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture);
			this.gl.texImage2D(
				this.gl.TEXTURE_2D,
				0,
				this.gl.RGBA,
				width,
				height,
				0,
				this.gl.RGBA,
				this.gl.UNSIGNED_BYTE,
				null,
			);
		}
		this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, 0);
		this.gl.pixelStorei(this.gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, 0);
		this.gl.activeTexture(this.gl.TEXTURE0);
		this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture);
		this.gl.texSubImage2D(this.gl.TEXTURE_2D, 0, 0, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, source);
		if (this.uploadedMaskRevision !== maskRevision) {
			this.gl.activeTexture(this.gl.TEXTURE1);
			this.gl.bindTexture(this.gl.TEXTURE_2D, this.maskTexture);
			this.gl.texSubImage2D(this.gl.TEXTURE_2D, 0, 0, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, mask);
			this.uploadedMaskRevision = maskRevision;
		}
		this.gl.useProgram(this.program);
		this.gl.bindVertexArray(this.vertexArray);
		this.gl.drawArrays(this.gl.TRIANGLES, 0, 3);
		this.gl.flush();
		return this.canvas;
	}

	dispose(): void {
		if (this.disposed) {
			return;
		}
		this.disposed = true;
		this.gl.deleteTexture(this.sourceTexture);
		this.gl.deleteTexture(this.maskTexture);
		this.gl.deleteVertexArray(this.vertexArray);
		this.gl.deleteProgram(this.program);
		this.gl.getExtension('WEBGL_lose_context')?.loseContext();
	}
}