import { continueRender, delayRender, RemotionVideoProps } from 'remotion'; import { GlslVarType, setUniform } from './glUtils'; export interface VideoShaderOptions { variables?: { type: GlslVarType; name: string; }[]; } export const ATTR_UV = 'a_uv'; export const ATTR_POS = 'a_pos'; export const VAR_UV = 'uv'; export const VAR_XY = 'xy'; const defaultVertexShader = ` precision highp float; attribute vec4 ${ATTR_POS}; attribute vec2 ${ATTR_UV}; varying vec2 ${VAR_UV}; varying vec2 ${VAR_XY}; uniform vec2 size; void main(void) { gl_Position = ${ATTR_POS}; ${VAR_UV} = ${ATTR_UV}; ${VAR_XY} = vec2( (${ATTR_POS}.x * 0.5 + 0.5) * float(size.x), (0.5 - ${ATTR_POS}.y * 0.5) * float(size.y) ); }`; export function createFragmentShader(content: string, options: VideoShaderOptions = {}) { const { variables } = options; return ` precision highp float; varying vec2 ${VAR_UV}; varying vec2 ${VAR_XY}; uniform vec2 size; uniform int frame; uniform float t; uniform sampler2D u_tex; ${!variables ? '' : variables.map(v => `uniform ${v.type} ${v.name};`).join('\n')} vec2 xyToUv(in vec2 _xy) { return vec2(_xy.x, _xy.y) / size; } void main(void) { ${content} }`; } export const defaultShader = createFragmentShader(`gl_FragColor = vec4(texture2D(u_tex, ${VAR_UV}).rgb, 1);`); // Example for animation using frame uniform variable inside shader // float r = radius * (sin(float(frame) * 0.02) * 0.5 + 0.5); export const swirlShader = createFragmentShader(` vec2 cUV = ${VAR_XY} - size / 2. - offset; float dist = length(cUV); if (dist < radius) { float percent = (radius - dist) / radius; float theta = percent * percent * angle * 8.0; float s = sin(theta); float c = cos(theta); cUV = vec2(dot(cUV, vec2(c, -s)), dot(cUV, vec2(s, c))); } cUV = xyToUv(cUV + offset + size / 2.); // / vec2(width, height) + vec2(0.5, 0.5); gl_FragColor = vec4(texture2D(u_tex, cUV).rgb, 0.5);`, { variables: [ { type: 'float', name: 'radius' }, { type: 'float', name: 'angle' }, { type: 'vec2', name: 'offset' }, ], } ); export interface ShaderVariable { name: string; type: GlslVarType; value: any; } export default class WebGlVideoProcessor { public readonly gl: WebGLRenderingContext; private _stop?: () => void; private _texture: WebGLTexture; private _program: WebGLProgram; private _vertexShader: WebGLShader; private _fragmentShader: WebGLShader; private _verticesBuffer: WebGLBuffer; private _uvBuffer: WebGLBuffer; private _delayRenderHandle?: number; private _videoLoaded = false; private _variables: ShaderVariable[] = []; private _frame: number = 0; constructor( public readonly canvas: HTMLCanvasElement, public readonly video: HTMLVideoElement, fragmentShader = defaultShader, vertexShader = defaultVertexShader ) { const gl = canvas.getContext('webgl', { preserveDrawingBuffer: true }); if (!gl) { throw new Error('Error initializing WebGl context'); } this.gl = gl; // Initialize resources this._texture = this._createTexture(); this._verticesBuffer = this._createBuffer([ 1.0, 1.0, 0.0, -1.0, 1.0, 0.0, // 1.0, -1.0, 0.0, -1.0, -1.0, 0.0 ]); this._uvBuffer = this._createBuffer([ 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0 ]); this._vertexShader = this._createShader(vertexShader, gl.VERTEX_SHADER); this._fragmentShader = this._createShader(fragmentShader, gl.FRAGMENT_SHADER); this._program = this._createProgram(); this.videoChanged(); } private delayRender() { if (!this._delayRenderHandle && this.isRunning) { this._delayRenderHandle = delayRender(); } } private continueRender() { if (this._delayRenderHandle) { continueRender(this._delayRenderHandle); this._delayRenderHandle = undefined; } } private _createBuffer(data: number[]) { const gl = this.gl; const buffer = gl.createBuffer(); if (!buffer) { throw new Error('Error creating vertex buffer'); } gl.bindBuffer(gl.ARRAY_BUFFER, buffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data), gl.STATIC_DRAW); return buffer; } private _createTexture() { const gl = this.gl; const texture = gl.createTexture(); if (!texture) { throw new Error('Error creating texture'); } gl.bindTexture(gl.TEXTURE_2D, texture); // Fill with yellow pixel gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, 1, 1, 0, gl.RGB, gl.UNSIGNED_BYTE, new Uint8Array([255, 255, 0])); // Set texture parameters gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); return texture; } private _createShader(source: string, type: number) { const gl = this.gl; const shader = gl.createShader(type); if (!shader) { throw new Error('Error creating shader'); } gl.shaderSource(shader, source); gl.compileShader(shader); if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { console.debug('Shader source:', source.split('\n').map((l, idx) => `${idx + 1}: ${l}`).join('\n')); throw new Error('Error compiling shader: ' + gl.getShaderInfoLog(shader)); } return shader; } private _createProgram() { if (!this._vertexShader || !this._fragmentShader) { throw new Error(); } const gl = this.gl; const program = gl.createProgram(); if (!program) { throw new Error('Error creating program'); } gl.attachShader(program, this._vertexShader); gl.attachShader(program, this._fragmentShader); // Link program gl.linkProgram(program); if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { throw new Error('Error linking shader program: ' + gl.getProgramInfoLog(program)); } // Set program parameters const posAttribute = gl.getAttribLocation(program, ATTR_POS); gl.bindBuffer(gl.ARRAY_BUFFER, this._verticesBuffer); gl.enableVertexAttribArray(posAttribute); gl.vertexAttribPointer(posAttribute, 3, gl.FLOAT, false, 0, 0); const uvAttribute = gl.getAttribLocation(program, ATTR_UV); gl.bindBuffer(gl.ARRAY_BUFFER, this._uvBuffer); gl.enableVertexAttribArray(uvAttribute); gl.vertexAttribPointer(uvAttribute, 2, gl.FLOAT, false, 0, 0); // Use new program gl.useProgram(program); return program; } public get isRunning() { return Boolean(this._stop); } public start() { let stopped = false; this._stop = () => { stopped = true; }; let lastTime = -1; const update = () => { if (stopped) { return; } if (this._videoLoaded) { // Check, if we actually have to update the frame if (lastTime !== this.video.currentTime) { lastTime = this.video.currentTime; this.render(); } // Continue render this.continueRender(); } window.requestAnimationFrame(update); }; window.requestAnimationFrame(update); } private render() { const gl = this.gl; const video = this.video; // Update video texture gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video); // Update variables for (const v of this._variables) { setUniform(gl, gl.getUniformLocation(this._program, v.name), v.type, v.value); } setUniform(gl, gl.getUniformLocation(this._program, 'frame'), 'int', this._frame); setUniform(gl, gl.getUniformLocation(this._program, 'size'), 'vec2', [video.videoWidth, video.videoHeight]); setUniform(gl, gl.getUniformLocation(this._program, 't'), 'float', video.currentTime); gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); } public stop() { this._stop?.(); this.continueRender(); } public videoChanged() { const gl = this.gl; const canvas = this.canvas; const video = this.video; this._videoLoaded = false; const delayRenderHandle = delayRender(); video.addEventListener('loadeddata', () => { // Set canvas size canvas.width = video.videoWidth; canvas.height = video.videoHeight; gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); continueRender(delayRenderHandle); this._videoLoaded = true; }, { once: true }); } public setFrame(frame: number) { this.delayRender(); this._frame = frame; } public setShader(fragmentShader = defaultShader, vertexShader = defaultVertexShader) { this._vertexShader = this._createShader(vertexShader, this.gl.VERTEX_SHADER); this._fragmentShader = this._createShader(fragmentShader, this.gl.FRAGMENT_SHADER); this._program = this._createProgram(); } public setVariables(variables: ShaderVariable[] | undefined) { this._variables = variables || []; } }