From 5a2b653129d494cad4c9dd77c6e832f4af0934e7 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Thu, 24 Sep 2020 22:50:17 +0200 Subject: [PATCH] Implement more advanced layer features for videos #54 --- README.md | 20 +++- examples/mosaic.json5 | 20 ++++ examples/resizeVertical.json5 | 14 --- examples/videos.json5 | 16 +-- examples/videos2.json5 | 14 +++ sources/frameSource.js | 1 + sources/videoFrameSource.js | 190 ++++++++++++++++++++++------------ 7 files changed, 187 insertions(+), 88 deletions(-) create mode 100644 examples/mosaic.json5 delete mode 100644 examples/resizeVertical.json5 create mode 100644 examples/videos2.json5 diff --git a/README.md b/README.md index 82022f9..648cd82 100644 --- a/README.md +++ b/README.md @@ -193,9 +193,15 @@ For video layers, if parent `clip.duration` is specified, the video will be slow | `resizeMode` | See [Resize modes](#resize-modes) | | | | `cutFrom` | Time value to cut from | `0` | sec | | `cutTo` | Time value to cut to | *end of video* | sec | -| `backgroundColor` | Background of letterboxing | `#000000` | | +| `width` | Width relative to screen width | `1` | `0` to `1` | +| `height` | Height relative to screen height | `1` | `0` to `1` | +| `left` | X-position relative to screen width | `0` | `0` to `1` | +| `top` | Y-position relative to screen height | `0` | `0` to `1` | +| `originX` | X anchor | `left` | `left` or `right` | +| `originY` | Y anchor | `top` | `top` or `bottom` | | `mixVolume` | Relative volume when mixing this video's audio track with others | `1` | | + #### Layer type 'audio' Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to values between `0.5x` and `100x`. @@ -302,9 +308,17 @@ Loads a GLSL shader. See [gl.json5](https://github.com/mifi/editly/blob/master/e ### Resize modes -`resizeMode` - How to fit image to screen. Can be one of `contain`, `contain-blur`, `cover`, `stretch`. Default `contain-blur`. +`resizeMode` - How to fit image to screen. Can be one of: +- `contain` - All the video will be contained within the frame and letterboxed +- `contain-blur` - Like `contain`, but with a blurred copy as the letterbox +- `cover` - Video be cropped to cover the whole screen (aspect ratio preserved) +- `stretch` - Video will be stretched to cover the whole screen (aspect ratio ignored). + +Default `contain-blur`. -See [image.json5](https://github.com/mifi/editly/blob/master/examples/image.json5) +See: +- [image.json5](https://github.com/mifi/editly/blob/master/examples/image.json5) +- [videos.json5](https://github.com/mifi/editly/blob/master/examples/videos.json5) ### Position parameter diff --git a/examples/mosaic.json5 b/examples/mosaic.json5 new file mode 100644 index 0000000..292104d --- /dev/null +++ b/examples/mosaic.json5 @@ -0,0 +1,20 @@ +{ + width: 500, height: 500, + outPath: './mosaic.mp4', + defaults: { + transition: { duration: 0 }, + layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' }, + layerType: { + video: { width: 0.4, height: 0.4 }, + } + }, + clips: [ + { duration: 2, layers: [ + { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'cover', top: 0.5, left: 0.5, originY: 'center', originX: 'center' }, + { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain' }, + { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur', left: 1, originX: 'right' }, + { type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur', left: 1, top: 1, originX: 'right', originY: 'bottom' }, + { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'stretch', top: 1, originY: 'bottom' }, + ] }, + ], +} \ No newline at end of file diff --git a/examples/resizeVertical.json5 b/examples/resizeVertical.json5 deleted file mode 100644 index 8495d46..0000000 --- a/examples/resizeVertical.json5 +++ /dev/null @@ -1,14 +0,0 @@ -{ - width: 240, height: 320, fps: 15, - outPath: './resizeVertical.mp4', - defaults: { - transition: { duration: 0 }, - layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' }, - }, - clips: [ - { duration: 2, layers: [{ type: 'title-background', text: 'Editly can handle all formats and sizes with different fits', background: { type: 'radial-gradient' } }] }, - { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain' }, { type: 'title', text: 'Contain' }] }, - { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'stretch' }, { type: 'title', text: 'Stretch' }] }, - { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title', text: 'Cover' }] }, - ], -} \ No newline at end of file diff --git a/examples/videos.json5 b/examples/videos.json5 index 5d7d0f5..92d12f4 100644 --- a/examples/videos.json5 +++ b/examples/videos.json5 @@ -1,14 +1,16 @@ { - // verbose: true, - // enableFfmpegLog: true, + width: 600, height: 800, outPath: './videos.mp4', defaults: { - transition: { - name: 'linearblur', - }, + transition: { duration: 0 }, + layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' }, }, clips: [ - { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title', text: 'Video 1' }] }, - { layers: [{ type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2 }] }, + { duration: 2, layers: [{ type: 'title-background', text: 'Editly can handle all formats and sizes with different fits', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain' }, { type: 'title', text: 'Contain' }] }, + { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur' }, { type: 'title', text: 'Contain (blur)' }] }, + { layers: [{ type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur' }, { type: 'title', text: 'Contain\n(blur, vertical)' }] }, + { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'stretch' }, { type: 'title', text: 'Stretch' }] }, + { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'cover' }, { type: 'title', text: 'Cover' }] }, ], } \ No newline at end of file diff --git a/examples/videos2.json5 b/examples/videos2.json5 new file mode 100644 index 0000000..1c1ae08 --- /dev/null +++ b/examples/videos2.json5 @@ -0,0 +1,14 @@ +{ + // verbose: true, + // enableFfmpegLog: true, + outPath: './video2.mp4', + defaults: { + transition: { + name: 'linearblur', + }, + }, + clips: [ + { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title', text: 'Video 1' }] }, + { layers: [{ type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2 }] }, + ], +} \ No newline at end of file diff --git a/sources/frameSource.js b/sources/frameSource.js index 285d9c4..41ae989 100644 --- a/sources/frameSource.js +++ b/sources/frameSource.js @@ -61,6 +61,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver if (logTimes) console.time('frameSource.readNextFrame'); const rgba = await frameSource.readNextFrame(offsetProgress, canvas); if (logTimes) console.timeEnd('frameSource.readNextFrame'); + // Frame sources can either render to the provided canvas and return nothing // OR return an raw RGBA blob which will be drawn onto the canvas if (rgba) { diff --git a/sources/videoFrameSource.js b/sources/videoFrameSource.js index fbe0336..cb5916f 100644 --- a/sources/videoFrameSource.js +++ b/sources/videoFrameSource.js @@ -3,15 +3,39 @@ const assert = require('assert'); const { getFfmpegCommonArgs } = require('../ffmpeg'); const { readFileStreams } = require('../util'); +const { rgbaToFabricImage, blurImage } = require('./fabric'); -module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => { - const targetSize = width * height * channels; +module.exports = async ({ width: canvasWidth, height: canvasHeight, channels, framerateStr, verbose, logTimes, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => { + const { path, cutFrom, cutTo, resizeMode = 'contain-blur', framePtsFactor, inputWidth, inputHeight, width: requestedWidthRel, height: requestedHeightRel, left: leftRel = 0, top: topRel = 0, originX = 'left', originY = 'top' } = params; - // TODO assert that we have read the correct amount of frames + const requestedWidth = requestedWidthRel ? requestedWidthRel * canvasWidth : canvasWidth; + const requestedHeight = requestedHeightRel ? requestedHeightRel * canvasHeight : canvasHeight; + + const left = leftRel * canvasWidth; + const top = topRel * canvasHeight; + + let targetWidth = requestedWidth; + let targetHeight = requestedHeight; + + if (['contain', 'contain-blur'].includes(resizeMode)) { + const ratioW = requestedWidth / inputWidth; + const ratioH = requestedHeight / inputHeight; + const inputAspectRatio = inputWidth / inputHeight; + + if (ratioW > ratioH) { + targetHeight = requestedHeight; + targetWidth = Math.round(requestedHeight * inputAspectRatio); + } else { + targetWidth = requestedWidth; + targetHeight = Math.round(requestedWidth / inputAspectRatio); + } + } - const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor } = params; + const frameByteSize = targetWidth * targetHeight * channels; - const buf = Buffer.allocUnsafe(targetSize); + // TODO assert that we have read the correct amount of frames + + const buf = Buffer.allocUnsafe(frameByteSize); let length = 0; // let inFrameCount = 0; @@ -22,11 +46,9 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg } let scaleFilter; - if (resizeMode === 'stretch') scaleFilter = `scale=${width}:${height}`; - // https://superuser.com/questions/891145/ffmpeg-upscale-and-letterbox-a-video/891478 - else if (resizeMode === 'contain' || resizeMode === 'contain-blur') scaleFilter = `scale=(iw*sar)*min(${width}/(iw*sar)\\,${height}/ih):ih*min(${width}/(iw*sar)\\,${height}/ih), pad=${width}:${height}:(${width}-iw*min(${width}/iw\\,${height}/ih))/2:(${height}-ih*min(${width}/iw\\,${height}/ih))/2:${backgroundColor}`; + if (['stretch', 'contain', 'contain-blur'].includes(resizeMode)) scaleFilter = `scale=${targetWidth}:${targetHeight}`; // Cover: https://unix.stackexchange.com/a/192123 - else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`; + else scaleFilter = `scale=(iw*sar)*max(${targetWidth}/(iw*sar)\\,${targetHeight}/ih):ih*max(${targetWidth}/(iw*sar)\\,${targetHeight}/ih),crop=${targetWidth}:${targetHeight}`; // https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/ const streams = await readFileStreams(ffprobePath, path); @@ -65,69 +87,109 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg ended = true; }); - const readNextFrame = () => new Promise((resolve, reject) => { - if (ended) { - console.log(path, 'Tried to read next video frame after ffmpeg video stream ended'); - resolve(); - return; - } - // console.log('Reading new frame', path); + async function readNextFrame(progress, canvas) { + const rgba = await new Promise((resolve, reject) => { + if (ended) { + console.log(path, 'Tried to read next video frame after ffmpeg video stream ended'); + resolve(); + return; + } + // console.log('Reading new frame', path); - function onEnd() { - resolve(); - } + function onEnd() { + resolve(); + } - function cleanup() { - stream.pause(); - // eslint-disable-next-line no-use-before-define - stream.removeListener('data', handleChunk); - stream.removeListener('end', onEnd); - stream.removeListener('error', reject); - } + function cleanup() { + stream.pause(); + // eslint-disable-next-line no-use-before-define + stream.removeListener('data', handleChunk); + stream.removeListener('end', onEnd); + stream.removeListener('error', reject); + } - function handleChunk(chunk) { - // console.log('chunk', chunk.length); - const nCopied = length + chunk.length > targetSize ? targetSize - length : chunk.length; - chunk.copy(buf, length, 0, nCopied); - length += nCopied; - - if (length > targetSize) console.error('Video data overflow', length); - - if (length >= targetSize) { - // console.log('Finished reading frame', inFrameCount, path); - const out = Buffer.from(buf); - - const restLength = chunk.length - nCopied; - if (restLength > 0) { - // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength); - chunk.slice(nCopied).copy(buf, 0); - length = restLength; - } else { - length = 0; - } + function handleChunk(chunk) { + // console.log('chunk', chunk.length); + const nCopied = length + chunk.length > frameByteSize ? frameByteSize - length : chunk.length; + chunk.copy(buf, length, 0, nCopied); + length += nCopied; - // inFrameCount += 1; + if (length > frameByteSize) console.error('Video data overflow', length); - clearTimeout(timeout); - cleanup(); - resolve(out); + if (length >= frameByteSize) { + // console.log('Finished reading frame', inFrameCount, path); + const out = Buffer.from(buf); + + const restLength = chunk.length - nCopied; + if (restLength > 0) { + // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength); + chunk.slice(nCopied).copy(buf, 0); + length = restLength; + } else { + length = 0; + } + + // inFrameCount += 1; + + clearTimeout(timeout); + cleanup(); + resolve(out); + } } + + timeout = setTimeout(() => { + console.warn('Timeout on read video frame'); + cleanup(); + resolve(); + }, 60000); + + stream.on('data', handleChunk); + stream.on('end', onEnd); + stream.on('error', reject); + stream.resume(); + }); + + if (!rgba) return; + + assert(rgba.length === frameByteSize); + + if (logTimes) console.time('rgbaToFabricImage'); + const img = await rgbaToFabricImage({ width: targetWidth, height: targetHeight, rgba }); + if (logTimes) console.timeEnd('rgbaToFabricImage'); + + img.setOptions({ + originX, + originY, + }); + + let centerOffsetX = 0; + let centerOffsetY = 0; + if (resizeMode === 'contain' || resizeMode === 'contain-blur') { + const dirX = originX === 'left' ? 1 : -1; + const dirY = originY === 'top' ? 1 : -1; + centerOffsetX = (dirX * (requestedWidth - targetWidth)) / 2; + centerOffsetY = (dirY * (requestedHeight - targetHeight)) / 2; } - timeout = setTimeout(() => { - console.warn('Timeout on read video frame'); - cleanup(); - resolve(); - }, 60000); - - stream.on('data', handleChunk); - stream.on('end', onEnd); - stream.on('error', reject); - stream.resume(); - }).then((data) => { - if (data) assert(data.length === targetSize); - return data; - }); + img.setOptions({ + left: left + centerOffsetX, + top: top + centerOffsetY, + }); + + if (resizeMode === 'contain-blur') { + const blurredImg = await new Promise((r) => img.cloneAsImage(r)); + blurImage({ mutableImg: blurredImg, width: requestedWidth, height: requestedHeight }); + blurredImg.setOptions({ + left, + top, + originX, + originY, + }); + canvas.add(blurredImg); + } + + canvas.add(img); + } const close = () => { if (verbose) console.log('Close', path);