Implement more advanced layer features for videos #54

README.md

@@ -193,9 +193,15 @@ For video layers, if parent `clip.duration` is specified, the video will be slow
 | `resizeMode` | See [Resize modes](#resize-modes) | | |
 | `cutFrom` | Time value to cut from | `0` | sec |
 | `cutTo` | Time value to cut to | *end of video* | sec |
-| `backgroundColor` | Background of letterboxing | `#000000` | |
+| `width` | Width relative to screen width | `1` | `0` to `1` |
+| `height` | Height relative to screen height | `1` | `0` to `1` |
+| `left` | X-position relative to screen width | `0` | `0` to `1` |
+| `top` | Y-position relative to screen height | `0` | `0` to `1` |
+| `originX` | X anchor | `left` | `left` or `right` |
+| `originY` | Y anchor | `top` | `top` or `bottom` |
 | `mixVolume` | Relative volume when mixing this video's audio track with others | `1` | |
 
+
 #### Layer type 'audio'
 
 Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to values between `0.5x` and `100x`.
@@ -302,9 +308,17 @@ Loads a GLSL shader. See [gl.json5](https://github.com/mifi/editly/blob/master/e
 
 ### Resize modes
 
-`resizeMode` - How to fit image to screen. Can be one of `contain`, `contain-blur`, `cover`, `stretch`. Default `contain-blur`.
+`resizeMode` - How to fit image to screen. Can be one of:
+- `contain` - All the video will be contained within the frame and letterboxed
+- `contain-blur` - Like `contain`, but with a blurred copy as the letterbox
+- `cover` - Video be cropped to cover the whole screen (aspect ratio preserved)
+- `stretch` - Video will be stretched to cover the whole screen (aspect ratio ignored).
+
+Default `contain-blur`.
 
-See [image.json5](https://github.com/mifi/editly/blob/master/examples/image.json5)
+See:
+- [image.json5](https://github.com/mifi/editly/blob/master/examples/image.json5)
+- [videos.json5](https://github.com/mifi/editly/blob/master/examples/videos.json5)
 
 ### Position parameter
 

examples/mosaic.json5

@@ -0,0 +1,20 @@
+{
+  width: 500, height: 500,
+  outPath: './mosaic.mp4',
+  defaults: {
+    transition: { duration: 0 },
+    layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' },
+    layerType: {
+      video: { width: 0.4, height: 0.4 },
+    }
+  },
+  clips: [
+    { duration: 2, layers: [
+      { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'cover', top: 0.5, left: 0.5, originY: 'center', originX: 'center' },
+      { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain' },
+      { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur', left: 1, originX: 'right' },
+      { type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur', left: 1, top: 1, originX: 'right', originY: 'bottom' },
+      { type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'stretch', top: 1, originY: 'bottom' },
+    ] },
+  ],
+}

examples/resizeVertical.json5

@@ -1,14 +0,0 @@
-{
-  width: 240, height: 320, fps: 15,
-  outPath: './resizeVertical.mp4',
-  defaults: {
-    transition: { duration: 0 },
-    layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' },
-  },
-  clips: [
-    { duration: 2, layers: [{ type: 'title-background', text: 'Editly can handle all formats and sizes with different fits', background: { type: 'radial-gradient' } }] },
-    { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain' }, { type: 'title', text: 'Contain' }] },
-    { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'stretch' }, { type: 'title', text: 'Stretch' }] },
-    { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title', text: 'Cover' }] },
-  ],
-}

examples/videos.json5

@@ -1,14 +1,16 @@
 {
-  // verbose: true,
-  // enableFfmpegLog: true,
+  width: 600, height: 800,
   outPath: './videos.mp4',
   defaults: {
-    transition: {
-      name: 'linearblur',
-    },
+    transition: { duration: 0 },
+    layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' },
   },
   clips: [
-    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title', text: 'Video 1' }] },
-    { layers: [{ type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2 }] },
+    { duration: 2, layers: [{ type: 'title-background', text: 'Editly can handle all formats and sizes with different fits', background: { type: 'radial-gradient' } }] },
+    { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain' }, { type: 'title', text: 'Contain' }] },
+    { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur' }, { type: 'title', text: 'Contain (blur)' }] },
+    { layers: [{ type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain-blur' }, { type: 'title', text: 'Contain\n(blur, vertical)' }] },
+    { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'stretch' }, { type: 'title', text: 'Stretch' }] },
+    { layers: [{ type: 'video', path: './assets/IMG_1322.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'cover' }, { type: 'title', text: 'Cover' }] },
   ],
 }

examples/videos2.json5

@@ -0,0 +1,14 @@
+{
+  // verbose: true,
+  // enableFfmpegLog: true,
+  outPath: './video2.mp4',
+  defaults: {
+    transition: {
+      name: 'linearblur',
+    },
+  },
+  clips: [
+    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title', text: 'Video 1' }] },
+    { layers: [{ type: 'video', path: './assets/IMG_1884.MOV', cutFrom: 0, cutTo: 2 }] },
+  ],
+}

sources/frameSource.js

@@ -61,6 +61,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
         if (logTimes) console.time('frameSource.readNextFrame');
         const rgba = await frameSource.readNextFrame(offsetProgress, canvas);
         if (logTimes) console.timeEnd('frameSource.readNextFrame');
+
         // Frame sources can either render to the provided canvas and return nothing
         // OR return an raw RGBA blob which will be drawn onto the canvas
         if (rgba) {

sources/videoFrameSource.js

@@ -3,15 +3,39 @@ const assert = require('assert');
 
 const { getFfmpegCommonArgs } = require('../ffmpeg');
 const { readFileStreams } = require('../util');
+const { rgbaToFabricImage, blurImage } = require('./fabric');
 
-module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
-  const targetSize = width * height * channels;
+module.exports = async ({ width: canvasWidth, height: canvasHeight, channels, framerateStr, verbose, logTimes, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
+  const { path, cutFrom, cutTo, resizeMode = 'contain-blur', framePtsFactor, inputWidth, inputHeight, width: requestedWidthRel, height: requestedHeightRel, left: leftRel = 0, top: topRel = 0, originX = 'left', originY = 'top' } = params;
 
-  // TODO assert that we have read the correct amount of frames
+  const requestedWidth = requestedWidthRel ? requestedWidthRel * canvasWidth : canvasWidth;
+  const requestedHeight = requestedHeightRel ? requestedHeightRel * canvasHeight : canvasHeight;
+
+  const left = leftRel * canvasWidth;
+  const top = topRel * canvasHeight;
+
+  let targetWidth = requestedWidth;
+  let targetHeight = requestedHeight;
+
+  if (['contain', 'contain-blur'].includes(resizeMode)) {
+    const ratioW = requestedWidth / inputWidth;
+    const ratioH = requestedHeight / inputHeight;
+    const inputAspectRatio = inputWidth / inputHeight;
+
+    if (ratioW > ratioH) {
+      targetHeight = requestedHeight;
+      targetWidth = Math.round(requestedHeight * inputAspectRatio);
+    } else {
+      targetWidth = requestedWidth;
+      targetHeight = Math.round(requestedWidth / inputAspectRatio);
+    }
+  }
 
-  const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor } = params;
+  const frameByteSize = targetWidth * targetHeight * channels;
 
-  const buf = Buffer.allocUnsafe(targetSize);
+  // TODO assert that we have read the correct amount of frames
+
+  const buf = Buffer.allocUnsafe(frameByteSize);
   let length = 0;
   // let inFrameCount = 0;
 
@@ -22,11 +46,9 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
   }
 
   let scaleFilter;
-  if (resizeMode === 'stretch') scaleFilter = `scale=${width}:${height}`;
-  // https://superuser.com/questions/891145/ffmpeg-upscale-and-letterbox-a-video/891478
-  else if (resizeMode === 'contain' || resizeMode === 'contain-blur') scaleFilter = `scale=(iw*sar)*min(${width}/(iw*sar)\\,${height}/ih):ih*min(${width}/(iw*sar)\\,${height}/ih), pad=${width}:${height}:(${width}-iw*min(${width}/iw\\,${height}/ih))/2:(${height}-ih*min(${width}/iw\\,${height}/ih))/2:${backgroundColor}`;
+  if (['stretch', 'contain', 'contain-blur'].includes(resizeMode)) scaleFilter = `scale=${targetWidth}:${targetHeight}`;
   // Cover: https://unix.stackexchange.com/a/192123
-  else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`;
+  else scaleFilter = `scale=(iw*sar)*max(${targetWidth}/(iw*sar)\\,${targetHeight}/ih):ih*max(${targetWidth}/(iw*sar)\\,${targetHeight}/ih),crop=${targetWidth}:${targetHeight}`;
 
   // https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/
   const streams = await readFileStreams(ffprobePath, path);
@@ -65,69 +87,109 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
     ended = true;
   });
 
-  const readNextFrame = () => new Promise((resolve, reject) => {
-    if (ended) {
-      console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
-      resolve();
-      return;
-    }
-    // console.log('Reading new frame', path);
+  async function readNextFrame(progress, canvas) {
+    const rgba = await new Promise((resolve, reject) => {
+      if (ended) {
+        console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
+        resolve();
+        return;
+      }
+      // console.log('Reading new frame', path);
 
-    function onEnd() {
-      resolve();
-    }
+      function onEnd() {
+        resolve();
+      }
 
-    function cleanup() {
-      stream.pause();
-      // eslint-disable-next-line no-use-before-define
-      stream.removeListener('data', handleChunk);
-      stream.removeListener('end', onEnd);
-      stream.removeListener('error', reject);
-    }
+      function cleanup() {
+        stream.pause();
+        // eslint-disable-next-line no-use-before-define
+        stream.removeListener('data', handleChunk);
+        stream.removeListener('end', onEnd);
+        stream.removeListener('error', reject);
+      }
 
-    function handleChunk(chunk) {
-      // console.log('chunk', chunk.length);
-      const nCopied = length + chunk.length > targetSize ? targetSize - length : chunk.length;
-      chunk.copy(buf, length, 0, nCopied);
-      length += nCopied;
-
-      if (length > targetSize) console.error('Video data overflow', length);
-
-      if (length >= targetSize) {
-        // console.log('Finished reading frame', inFrameCount, path);
-        const out = Buffer.from(buf);
-
-        const restLength = chunk.length - nCopied;
-        if (restLength > 0) {
-          // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
-          chunk.slice(nCopied).copy(buf, 0);
-          length = restLength;
-        } else {
-          length = 0;
-        }
+      function handleChunk(chunk) {
+        // console.log('chunk', chunk.length);
+        const nCopied = length + chunk.length > frameByteSize ? frameByteSize - length : chunk.length;
+        chunk.copy(buf, length, 0, nCopied);
+        length += nCopied;
 
-        // inFrameCount += 1;
+        if (length > frameByteSize) console.error('Video data overflow', length);
 
-        clearTimeout(timeout);
-        cleanup();
-        resolve(out);
+        if (length >= frameByteSize) {
+          // console.log('Finished reading frame', inFrameCount, path);
+          const out = Buffer.from(buf);
+
+          const restLength = chunk.length - nCopied;
+          if (restLength > 0) {
+            // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
+            chunk.slice(nCopied).copy(buf, 0);
+            length = restLength;
+          } else {
+            length = 0;
+          }
+
+          // inFrameCount += 1;
+
+          clearTimeout(timeout);
+          cleanup();
+          resolve(out);
+        }
       }
+
+      timeout = setTimeout(() => {
+        console.warn('Timeout on read video frame');
+        cleanup();
+        resolve();
+      }, 60000);
+
+      stream.on('data', handleChunk);
+      stream.on('end', onEnd);
+      stream.on('error', reject);
+      stream.resume();
+    });
+
+    if (!rgba) return;
+
+    assert(rgba.length === frameByteSize);
+
+    if (logTimes) console.time('rgbaToFabricImage');
+    const img = await rgbaToFabricImage({ width: targetWidth, height: targetHeight, rgba });
+    if (logTimes) console.timeEnd('rgbaToFabricImage');
+
+    img.setOptions({
+      originX,
+      originY,
+    });
+
+    let centerOffsetX = 0;
+    let centerOffsetY = 0;
+    if (resizeMode === 'contain' || resizeMode === 'contain-blur') {
+      const dirX = originX === 'left' ? 1 : -1;
+      const dirY = originY === 'top' ? 1 : -1;
+      centerOffsetX = (dirX * (requestedWidth - targetWidth)) / 2;
+      centerOffsetY = (dirY * (requestedHeight - targetHeight)) / 2;
     }
 
-    timeout = setTimeout(() => {
-      console.warn('Timeout on read video frame');
-      cleanup();
-      resolve();
-    }, 60000);
-
-    stream.on('data', handleChunk);
-    stream.on('end', onEnd);
-    stream.on('error', reject);
-    stream.resume();
-  }).then((data) => {
-    if (data) assert(data.length === targetSize);
-    return data;
-  });
+    img.setOptions({
+      left: left + centerOffsetX,
+      top: top + centerOffsetY,
+    });
+
+    if (resizeMode === 'contain-blur') {
+      const blurredImg = await new Promise((r) => img.cloneAsImage(r));
+      blurImage({ mutableImg: blurredImg, width: requestedWidth, height: requestedHeight });
+      blurredImg.setOptions({
+        left,
+        top,
+        originX,
+        originY,
+      });
+      canvas.add(blurredImg);
+    }
+
+    canvas.add(img);
+  }
 
   const close = () => {
     if (verbose) console.log('Close', path);