2020-2021-Pair-test-1-Tp-test
/
Michel-Beaussart-Main-git-repo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Improve layer rendering #47 

By reusing a fabric canvas for all fabric layers to render on.
pull/64/head
Mikael Finstad 6 years ago
parent
adb90227dc
commit
019731f90a
3 changed files with 74 additions and 73 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 106
      sources/fabricFrameSource.js
  2. 38
      sources/frameSource.js
  3. 3
      sources/videoFrameSource.js

106
sources/fabricFrameSource.js
View File

@ -23,58 +23,40 @@ function fabricCanvasToRgba(canvas) {
return canvasToRgba(ctx);
}
async function mergeFrames({ width, height, framesRaw }) {
if (framesRaw.length === 1) return framesRaw[0];
function createFabricCanvas({ width, height }) {
return new fabric.StaticCanvas(null, { width, height });
}
async function renderFabricCanvas(canvas) {
canvas.renderAll();
const rgba = fabricCanvasToRgba(canvas);
canvas.clear();
// canvas.dispose();
return rgba;
}
// Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668
async function rgbaToFabricImage({ width, height, rgba }) {
const canvas = createCanvas(width, height);
const ctx = canvas.getContext('2d');
framesRaw.forEach((frameRaw) => {
const canvas2 = createCanvas(width, height);
const ctx2 = canvas2.getContext('2d');
// https://developer.mozilla.org/en-US/docs/Web/API/ImageData/ImageData
// https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/putImageData
ctx2.putImageData(new nodeCanvas.ImageData(Uint8ClampedArray.from(frameRaw), width, height), 0, 0);
// require('fs').writeFileSync(`${Math.floor(Math.random() * 1e12)}.png`, canvas2.toBuffer('image/png'));
ctx.drawImage(canvas2, 0, 0);
});
return canvasToRgba(ctx);
// https://developer.mozilla.org/en-US/docs/Web/API/ImageData/ImageData
// https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/putImageData
ctx.putImageData(new nodeCanvas.ImageData(Uint8ClampedArray.from(rgba), width, height), 0, 0);
// https://stackoverflow.com/questions/58209996/unable-to-render-tiff-images-and-add-it-as-a-fabric-object
return new fabric.Image(canvas);
}
async function createFabricFrameSource(func, { width, height, ...rest }) {
const onInit = async ({ canvas }) => func(({ width, height, fabric, canvas, ...rest }));
let canvas = new fabric.StaticCanvas(null, { width, height });
const { onRender = () => {}, onClose = () => {} } = await onInit({ canvas }) || {};
async function readNextFrame(progress) {
await onRender(progress);
canvas.renderAll();
const rgba = fabricCanvasToRgba(canvas);
const onInit = async () => func(({ width, height, fabric, ...rest }));
canvas.clear();
// canvas.dispose();
return rgba;
}
const { onRender = () => {}, onClose = () => {} } = await onInit() || {};
return {
readNextFrame,
close: () => {
// https://stackoverflow.com/questions/19030174/how-to-manage-memory-in-case-of-multiple-fabric-js-canvas
canvas.dispose();
canvas = undefined;
onClose();
},
readNextFrame: onRender,
close: onClose,
};
}
async function imageFrameSource({ verbose, params, width, height, canvas }) {
async function imageFrameSource({ verbose, params, width, height }) {
if (verbose) console.log('Loading', params.path);
const imgData = await new Promise((resolve) => fabric.util.loadImage(fileUrl(params.path), resolve));
@ -95,7 +77,7 @@ async function imageFrameSource({ verbose, params, width, height, canvas }) {
else blurredImg.scaleToHeight(height);
async function onRender(progress) {
async function onRender(progress, canvas) {
const { zoomDirection = 'in', zoomAmount = 0.1 } = params;
const img = getImg();
@ -119,14 +101,20 @@ async function imageFrameSource({ verbose, params, width, height, canvas }) {
return { onRender, onClose };
}
async function fillColorFrameSource({ canvas, params }) {
async function fillColorFrameSource({ params, width, height }) {
const { color } = params;
const randomColor = getRandomColors(1)[0];
async function onRender() {
// eslint-disable-next-line no-param-reassign
canvas.backgroundColor = color || randomColor;
async function onRender(progress, canvas) {
const rect = new fabric.Rect({
left: 0,
right: 0,
width,
height,
fill: color || randomColor,
});
canvas.add(rect);
}
return { onRender };
@ -137,12 +125,12 @@ function getRekt(width, height) {
return new fabric.Rect({ originX: 'center', originY: 'center', left: width / 2, top: height / 2, width: width * 2, height: height * 2 });
}
async function radialGradientFrameSource({ canvas, width, height, params }) {
async function radialGradientFrameSource({ width, height, params }) {
const { colors: inColors } = params;
const randomColors = getRandomGradient();
async function onRender(progress) {
async function onRender(progress, canvas) {
// console.log('progress', progress);
const max = Math.max(width, height);
@ -177,13 +165,13 @@ async function radialGradientFrameSource({ canvas, width, height, params }) {
return { onRender };
}
async function linearGradientFrameSource({ canvas, width, height, params }) {
async function linearGradientFrameSource({ width, height, params }) {
const { colors: inColors } = params;
const randomColors = getRandomGradient();
const colors = inColors && inColors.length === 2 ? inColors : randomColors;
async function onRender(progress) {
async function onRender(progress, canvas) {
const rect = getRekt(width, height);
rect.setGradient('fill', {
@ -204,11 +192,11 @@ async function linearGradientFrameSource({ canvas, width, height, params }) {
return { onRender };
}
async function subtitleFrameSource({ canvas, width, height, params }) {
const { text, textColor = '#ffffff', fontFamily = 'sans-serif' } = params;
async function subtitleFrameSource({ width, height, params }) {
const { text, textColor = '#ffffff', backgroundColor = 'rgba(0,0,0,0.3)', fontFamily = 'sans-serif', delay = 0, speed = 1 } = params;
async function onRender(progress) {
const easedProgress = easeOutExpo(Math.min(progress, 1));
async function onRender(progress, canvas) {
const easedProgress = easeOutExpo(Math.max(0, Math.min((progress - delay) * speed, 1)));
const min = Math.min(width, height);
const padding = 0.05 * min;
@ -233,7 +221,7 @@ async function subtitleFrameSource({ canvas, width, height, params }) {
height: textBox.height + padding * 2,
top: height,
originY: 'bottom',
fill: 'rgba(0,0,0,0.2)',
fill: backgroundColor,
opacity: easedProgress,
});
@ -244,10 +232,10 @@ async function subtitleFrameSource({ canvas, width, height, params }) {
return { onRender };
}
async function titleFrameSource({ canvas, width, height, params }) {
async function titleFrameSource({ width, height, params }) {
const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center' } = params;
async function onRender(progress) {
async function onRender(progress, canvas) {
// console.log('progress', progress);
const min = Math.min(width, height);
@ -300,6 +288,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) {
context.clearRect(0, 0, canvas.width, canvas.height);
await onRender(progress);
// require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png'));
// I don't know any way to draw a node-canvas as a layer on a fabric.js canvas, other than converting to rgba first:
return canvasToRgba(context);
}
@ -319,7 +308,6 @@ function registerFont(...args) {
}
module.exports = {
mergeFrames,
registerFont,
createFabricFrameSource,
createCustomCanvasFrameSource,
@ -331,4 +319,8 @@ module.exports = {
radialGradientFrameSource,
linearGradientFrameSource,
imageFrameSource,
createFabricCanvas,
renderFabricCanvas,
rgbaToFabricImage,
};

38
sources/frameSource.js
View File

@ -1,7 +1,7 @@
const assert = require('assert');
const pMap = require('p-map');
const { mergeFrames, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource } = require('./fabricFrameSource');
const { rgbaToFabricImage, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource, createFabricCanvas, renderFabricCanvas } = require('./fabricFrameSource');
const createVideoFrameSource = require('./videoFrameSource');
const { createGlFrameSource } = require('./glFrameSource');
@ -9,7 +9,7 @@ const { createGlFrameSource } = require('./glFrameSource');
async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, enableFfmpegLog, framerateStr }) {
const { layers, duration } = clip;
const frameSources = await pMap(layers, async (layer, layerIndex) => {
const layerFrameSources = await pMap(layers, async (layer, layerIndex) => {
const { type, ...params } = layer;
console.log('createFrameSource', type, 'clip', clipIndex, 'layer', layerIndex);
@ -25,29 +25,37 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
'radial-gradient': async (opts) => createFabricFrameSource(radialGradientFrameSource, opts),
'fill-color': async (opts) => createFabricFrameSource(fillColorFrameSource, opts),
};
assert(frameSourceFuncs[type], `Invalid type ${type}`);
const createFrameSourceFunc = frameSourceFuncs[type];
assert(createFrameSourceFunc, `Invalid type ${type}`);
return createFrameSourceFunc({ ffmpegPath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
}, { concurrency: 1 });
async function readNextFrame(...args) {
const framesRaw = await pMap(frameSources, async (frameSource) => frameSource.readNextFrame(...args));
async function readNextFrame(progress) {
const canvas = createFabricCanvas({ width, height });
// eslint-disable-next-line no-restricted-syntax
for (const frameSource of layerFrameSources) {
const rgba = await frameSource.readNextFrame(progress, canvas);
// Frame sources can either render to the provided canvas and return nothing
// OR return an raw RGBA blob which will be drawn onto the canvas
if (rgba) {
// Optimization: Don't need to draw to canvas if there's only one layer
if (layerFrameSources.length === 1) return rgba;
const img = await rgbaToFabricImage({ width, height, rgba });
canvas.add(img);
} else {
// Assume this frame source has drawn its content to the canvas
}
}
// if (verbose) console.time('Merge frames');
const framesRawFiltered = framesRaw.filter((frameRaw) => {
if (frameRaw) return true;
console.warn('Frame source returned empty result');
return false;
});
const merged = mergeFrames({ width, height, framesRaw: framesRawFiltered });
// if (verbose) console.timeEnd('Merge frames');
return merged;
return renderFabricCanvas(canvas);
}
async function close() {
await pMap(frameSources, async (frameSource) => frameSource.close());
await pMap(layerFrameSources, async (frameSource) => frameSource.close());
}
return {

3
sources/videoFrameSource.js
View File

@ -51,13 +51,14 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
const readNextFrame = () => new Promise((resolve, reject) => {
if (ended) {
console.log(path, 'Tried to read next video frame after ffmpeg stream ended');
resolve();
return;
}
// console.log('Reading new frame', path);
function onEnd() {
if (verbose) console.log(path, 'ffmpeg stream ended');
if (verbose) console.log(path, 'ffmpeg video stream ended');
ended = true;
resolve();
}

Write Preview
Loading…
Cancel
Save