From 019731f90a876d50b02118500653cdd92aaa4908 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 14 Jul 2020 14:34:22 +0200 Subject: [PATCH] Improve layer rendering #47 By reusing a fabric canvas for all fabric layers to render on. --- sources/fabricFrameSource.js | 106 ++++++++++++++++------------------- sources/frameSource.js | 38 ++++++++----- sources/videoFrameSource.js | 3 +- 3 files changed, 74 insertions(+), 73 deletions(-) diff --git a/sources/fabricFrameSource.js b/sources/fabricFrameSource.js index 18cb4fd..d8452f2 100644 --- a/sources/fabricFrameSource.js +++ b/sources/fabricFrameSource.js @@ -23,58 +23,40 @@ function fabricCanvasToRgba(canvas) { return canvasToRgba(ctx); } -async function mergeFrames({ width, height, framesRaw }) { - if (framesRaw.length === 1) return framesRaw[0]; +function createFabricCanvas({ width, height }) { + return new fabric.StaticCanvas(null, { width, height }); +} + +async function renderFabricCanvas(canvas) { + canvas.renderAll(); + const rgba = fabricCanvasToRgba(canvas); + canvas.clear(); + // canvas.dispose(); + return rgba; +} - // Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668 +async function rgbaToFabricImage({ width, height, rgba }) { const canvas = createCanvas(width, height); const ctx = canvas.getContext('2d'); - - framesRaw.forEach((frameRaw) => { - const canvas2 = createCanvas(width, height); - const ctx2 = canvas2.getContext('2d'); - // https://developer.mozilla.org/en-US/docs/Web/API/ImageData/ImageData - // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/putImageData - ctx2.putImageData(new nodeCanvas.ImageData(Uint8ClampedArray.from(frameRaw), width, height), 0, 0); - // require('fs').writeFileSync(`${Math.floor(Math.random() * 1e12)}.png`, canvas2.toBuffer('image/png')); - - ctx.drawImage(canvas2, 0, 0); - }); - - return canvasToRgba(ctx); + // https://developer.mozilla.org/en-US/docs/Web/API/ImageData/ImageData + // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/putImageData + ctx.putImageData(new nodeCanvas.ImageData(Uint8ClampedArray.from(rgba), width, height), 0, 0); + // https://stackoverflow.com/questions/58209996/unable-to-render-tiff-images-and-add-it-as-a-fabric-object + return new fabric.Image(canvas); } async function createFabricFrameSource(func, { width, height, ...rest }) { - const onInit = async ({ canvas }) => func(({ width, height, fabric, canvas, ...rest })); - - let canvas = new fabric.StaticCanvas(null, { width, height }); - - const { onRender = () => {}, onClose = () => {} } = await onInit({ canvas }) || {}; - - async function readNextFrame(progress) { - await onRender(progress); - - canvas.renderAll(); - - const rgba = fabricCanvasToRgba(canvas); + const onInit = async () => func(({ width, height, fabric, ...rest })); - canvas.clear(); - // canvas.dispose(); - return rgba; - } + const { onRender = () => {}, onClose = () => {} } = await onInit() || {}; return { - readNextFrame, - close: () => { - // https://stackoverflow.com/questions/19030174/how-to-manage-memory-in-case-of-multiple-fabric-js-canvas - canvas.dispose(); - canvas = undefined; - onClose(); - }, + readNextFrame: onRender, + close: onClose, }; } -async function imageFrameSource({ verbose, params, width, height, canvas }) { +async function imageFrameSource({ verbose, params, width, height }) { if (verbose) console.log('Loading', params.path); const imgData = await new Promise((resolve) => fabric.util.loadImage(fileUrl(params.path), resolve)); @@ -95,7 +77,7 @@ async function imageFrameSource({ verbose, params, width, height, canvas }) { else blurredImg.scaleToHeight(height); - async function onRender(progress) { + async function onRender(progress, canvas) { const { zoomDirection = 'in', zoomAmount = 0.1 } = params; const img = getImg(); @@ -119,14 +101,20 @@ async function imageFrameSource({ verbose, params, width, height, canvas }) { return { onRender, onClose }; } -async function fillColorFrameSource({ canvas, params }) { +async function fillColorFrameSource({ params, width, height }) { const { color } = params; const randomColor = getRandomColors(1)[0]; - async function onRender() { - // eslint-disable-next-line no-param-reassign - canvas.backgroundColor = color || randomColor; + async function onRender(progress, canvas) { + const rect = new fabric.Rect({ + left: 0, + right: 0, + width, + height, + fill: color || randomColor, + }); + canvas.add(rect); } return { onRender }; @@ -137,12 +125,12 @@ function getRekt(width, height) { return new fabric.Rect({ originX: 'center', originY: 'center', left: width / 2, top: height / 2, width: width * 2, height: height * 2 }); } -async function radialGradientFrameSource({ canvas, width, height, params }) { +async function radialGradientFrameSource({ width, height, params }) { const { colors: inColors } = params; const randomColors = getRandomGradient(); - async function onRender(progress) { + async function onRender(progress, canvas) { // console.log('progress', progress); const max = Math.max(width, height); @@ -177,13 +165,13 @@ async function radialGradientFrameSource({ canvas, width, height, params }) { return { onRender }; } -async function linearGradientFrameSource({ canvas, width, height, params }) { +async function linearGradientFrameSource({ width, height, params }) { const { colors: inColors } = params; const randomColors = getRandomGradient(); const colors = inColors && inColors.length === 2 ? inColors : randomColors; - async function onRender(progress) { + async function onRender(progress, canvas) { const rect = getRekt(width, height); rect.setGradient('fill', { @@ -204,11 +192,11 @@ async function linearGradientFrameSource({ canvas, width, height, params }) { return { onRender }; } -async function subtitleFrameSource({ canvas, width, height, params }) { - const { text, textColor = '#ffffff', fontFamily = 'sans-serif' } = params; +async function subtitleFrameSource({ width, height, params }) { + const { text, textColor = '#ffffff', backgroundColor = 'rgba(0,0,0,0.3)', fontFamily = 'sans-serif', delay = 0, speed = 1 } = params; - async function onRender(progress) { - const easedProgress = easeOutExpo(Math.min(progress, 1)); + async function onRender(progress, canvas) { + const easedProgress = easeOutExpo(Math.max(0, Math.min((progress - delay) * speed, 1))); const min = Math.min(width, height); const padding = 0.05 * min; @@ -233,7 +221,7 @@ async function subtitleFrameSource({ canvas, width, height, params }) { height: textBox.height + padding * 2, top: height, originY: 'bottom', - fill: 'rgba(0,0,0,0.2)', + fill: backgroundColor, opacity: easedProgress, }); @@ -244,10 +232,10 @@ async function subtitleFrameSource({ canvas, width, height, params }) { return { onRender }; } -async function titleFrameSource({ canvas, width, height, params }) { +async function titleFrameSource({ width, height, params }) { const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center' } = params; - async function onRender(progress) { + async function onRender(progress, canvas) { // console.log('progress', progress); const min = Math.min(width, height); @@ -300,6 +288,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) { context.clearRect(0, 0, canvas.width, canvas.height); await onRender(progress); // require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png')); + // I don't know any way to draw a node-canvas as a layer on a fabric.js canvas, other than converting to rgba first: return canvasToRgba(context); } @@ -319,7 +308,6 @@ function registerFont(...args) { } module.exports = { - mergeFrames, registerFont, createFabricFrameSource, createCustomCanvasFrameSource, @@ -331,4 +319,8 @@ module.exports = { radialGradientFrameSource, linearGradientFrameSource, imageFrameSource, + + createFabricCanvas, + renderFabricCanvas, + rgbaToFabricImage, }; diff --git a/sources/frameSource.js b/sources/frameSource.js index cdfe545..aff727f 100644 --- a/sources/frameSource.js +++ b/sources/frameSource.js @@ -1,7 +1,7 @@ const assert = require('assert'); const pMap = require('p-map'); -const { mergeFrames, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource } = require('./fabricFrameSource'); +const { rgbaToFabricImage, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource, createFabricCanvas, renderFabricCanvas } = require('./fabricFrameSource'); const createVideoFrameSource = require('./videoFrameSource'); const { createGlFrameSource } = require('./glFrameSource'); @@ -9,7 +9,7 @@ const { createGlFrameSource } = require('./glFrameSource'); async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, enableFfmpegLog, framerateStr }) { const { layers, duration } = clip; - const frameSources = await pMap(layers, async (layer, layerIndex) => { + const layerFrameSources = await pMap(layers, async (layer, layerIndex) => { const { type, ...params } = layer; console.log('createFrameSource', type, 'clip', clipIndex, 'layer', layerIndex); @@ -25,29 +25,37 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver 'radial-gradient': async (opts) => createFabricFrameSource(radialGradientFrameSource, opts), 'fill-color': async (opts) => createFabricFrameSource(fillColorFrameSource, opts), }; - assert(frameSourceFuncs[type], `Invalid type ${type}`); - const createFrameSourceFunc = frameSourceFuncs[type]; + assert(createFrameSourceFunc, `Invalid type ${type}`); return createFrameSourceFunc({ ffmpegPath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); }, { concurrency: 1 }); - async function readNextFrame(...args) { - const framesRaw = await pMap(frameSources, async (frameSource) => frameSource.readNextFrame(...args)); + async function readNextFrame(progress) { + const canvas = createFabricCanvas({ width, height }); + + // eslint-disable-next-line no-restricted-syntax + for (const frameSource of layerFrameSources) { + const rgba = await frameSource.readNextFrame(progress, canvas); + // Frame sources can either render to the provided canvas and return nothing + // OR return an raw RGBA blob which will be drawn onto the canvas + if (rgba) { + // Optimization: Don't need to draw to canvas if there's only one layer + if (layerFrameSources.length === 1) return rgba; + + const img = await rgbaToFabricImage({ width, height, rgba }); + canvas.add(img); + } else { + // Assume this frame source has drawn its content to the canvas + } + } // if (verbose) console.time('Merge frames'); - const framesRawFiltered = framesRaw.filter((frameRaw) => { - if (frameRaw) return true; - console.warn('Frame source returned empty result'); - return false; - }); - const merged = mergeFrames({ width, height, framesRaw: framesRawFiltered }); - // if (verbose) console.timeEnd('Merge frames'); - return merged; + return renderFabricCanvas(canvas); } async function close() { - await pMap(frameSources, async (frameSource) => frameSource.close()); + await pMap(layerFrameSources, async (frameSource) => frameSource.close()); } return { diff --git a/sources/videoFrameSource.js b/sources/videoFrameSource.js index 6678636..f697ac0 100644 --- a/sources/videoFrameSource.js +++ b/sources/videoFrameSource.js @@ -51,13 +51,14 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg const readNextFrame = () => new Promise((resolve, reject) => { if (ended) { + console.log(path, 'Tried to read next video frame after ffmpeg stream ended'); resolve(); return; } // console.log('Reading new frame', path); function onEnd() { - if (verbose) console.log(path, 'ffmpeg stream ended'); + if (verbose) console.log(path, 'ffmpeg video stream ended'); ended = true; resolve(); }