Merge branch 'master' of https://github.com/mifi/editly

README.md

@@ -4,7 +4,7 @@
 
 This GIF / YouTube was created with this command: "editly [commonFeatures.json5](https://github.com/mifi/editly/blob/master/examples/commonFeatures.json5)". See [more examples here](https://github.com/mifi/editly/tree/master/examples#examples).
 
-**Editly** is a tool and framework for declarative NLE (**non-linear video editing**) using Node.js and ffmpeg. Editly allows you to easily and **programmatically create a video** from a **set of clips, images and titles**, with smooth transitions and music overlaid.
+**Editly** is a tool and framework for declarative NLE (**non-linear video editing**) using Node.js and ffmpeg. Editly allows you to easily and **programmatically create a video** from a **set of clips, images, audio and titles**, with smooth transitions and music overlaid.
 
 Editly has a simple CLI for quickly assembling a video from a set of clips or images, or you can use its more flexible JavaScript API.
 
@@ -22,12 +22,16 @@ Inspired by [ffmpeg-concat](https://github.com/transitive-bullshit/ffmpeg-concat
 - Accepts custom HTML5 Canvas / Fabric.js JavaScript code for custom screens or dynamic overlays
 - Render custom GL shaders (for example from [shadertoy](https://www.shadertoy.com/))
 - Can output GIF
+- Preserve audio sources or mix multiple
+- Overlay transparent images or videos
+- Show different sub-clips for parts of a clips duration (B-roll)
 
 ## Use cases
 
 - Create a slideshow from a set of pictures with text overlay
 - Create a fast-paced trailer or promo video
 - Create a tutorial video with help text
+- Create news stories
 - Simply convert a video to a GIF
 - Resize video to any size or framerate and with automatic letterboxing/cropping (e.g. if you need to upload a video somewhere but the site complains `Video must be 1337x1000 30fps`)
 
@@ -110,9 +114,16 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit
     layer: {
       fontPath,
       // ...more layer defaults
-    }
+    },
+    layerType: {
+      'fill-color': {
+        color: '#ff6666',
+      }
+      // ...more per-layer-type defaults
+    },
   },
   audioFilePath,
+  keepSourceAudio: false,
   clips: [
     {
       transition,
@@ -146,21 +157,24 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit
 | `width` | `--width` | Width which all media will be converted to | `640` | |
 | `height` | `--height` | Height which all media will be converted to | auto based on `width` and aspect ratio of **first video** | |
 | `fps` | `--fps` | FPS which all videos will be converted to | First video FPS or `25` | |
-| `audioFilePath` | `--audio-file-path` | Set an audio track to the whole output video | | |
+| `audioFilePath` | `--audio-file-path` | Set an audio track for the whole video | | |
+| `keepSourceAudio` | `--keep-source-audio` | Keep audio from source files | | |
 | `fast` | `--fast`, `-f` | Fast mode (low resolution and FPS, useful for getting a quick preview) | `false` | |
 | `defaults.layer.fontPath` | `--font-path` | Set default font to a .ttf | System font | |
 | `defaults.layer.*` | | Set any layer parameter that all layers will inherit | | |
 | `defaults.duration` | `--clip-duration` | Set default clip duration for clips that don't have an own duration | `4` | sec |
 | `defaults.transition` | | An object `{ name, duration }` describing the default transition. Set to **null** to disable transitions | | |
 | `defaults.transition.duration` | `--transition-duration` | Default transition duration | `0.5` | sec |
-| `defaults.transition.name` | `--transition-name` | Default transition type. See **Transition types** | `random` | |
-| `clips[]` | | List of clip objects that will be concatenated in sequence | | |
-| `clips[].duration` | | Clip duration. See `defaults.duration` | `defaults.duration` | |
+| `defaults.transition.name` | `--transition-name` | Default transition type. See [Transition types](#transition-types) | `random` | |
+| `clips[]` | | List of clip objects that will be played in sequence. Each clip can have one or more layers. | | |
+| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the **first video layer**. | `defaults.duration` | |
 | `clips[].transition` | | Specify transition at the **end** of this clip. See `defaults.transition` | `defaults.transition` | |
-| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (last layer on top) | | |
+| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (final layer on top) | | |
 | `clips[].layers[].type` | | Layer type, see below | | |
 | `onStart` | | Callback function, called before ffmpeg process start with command line as string argument | | |
 | `onProcessStart` | | Callback function, called after ffmpeg process start with subprocess argument | | |
+| `clips[].layers[].visibleFrom` | | What time into the clip should this layer start | | sec |
+| `clips[].layers[].visibleUntil` | | What time into the clip should this layer stop | | sec |
 
 ### Transition types
 
@@ -172,29 +186,56 @@ See [examples](https://github.com/mifi/editly/tree/master/examples) and [commonF
 
 #### Layer type 'video'
 
-For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`.
+For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. If the layer has audio, it will be kept (and mixed with other audio layers if present.)
 
 | Parameter  | Description | Default | |
 |-|-|-|-|
 | `path` | Path to video file | | |
 | `resizeMode` | One of `cover`, `contain`, `stretch` | `contain` | |
 | `cutFrom` | Time value to cut from | `0` | sec |
-| `cutTo` | Time value to cut from | *end of video* | sec |
+| `cutTo` | Time value to cut to | *end of video* | sec |
 | `backgroundColor` | Background of letterboxing | `#000000` | |
+| `mixVolume` | Relative volume when mixing this video's audio track with others | `1` | |
+
+#### Layer type 'audio'
+
+Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to values between `0.5x` and `100x`.
+
+| Parameter  | Description | Default | |
+|-|-|-|-|
+| `path` | Path to audio file | | |
+| `cutFrom` | Time value to cut from | `0` | sec |
+| `cutTo` | Time value to cut to | `clip.duration` | sec |
+| `mixVolume` | Relative volume when mixing this audio track with others | `1` | |
 
 #### Layer type 'image'
 
+Full screen image (auto letterboxed)
+
 | Parameter  | Description | Default | |
 |-|-|-|-|
 | `path` | Path to image file | | |
-| `zoomDirection` | Zoom direction for Ken Burns effect: `in`, `out` or `null` to disable | `in` | |
-| `zoomAmount` | Zoom amount for Ken Burns effect | `0.1` | |
+
+See also See [Ken Burns parameters](#ken-burns-parameters).
+
+#### Layer type 'image-overlay'
+
+Image overlay with a custom position on the screen.
+
+| Parameter  | Description | Default | |
+|-|-|-|-|
+| `path` | Path to image file | | |
+| `position` | See [Position parameter](#position-parameter) | | |
+
+See also [Ken Burns parameters](#ken-burns-parameters).
 
 #### Layer type 'title'
 - `fontPath` - See `defaults.layer.fontPath`
 - `text` - Title text to show, keep it short
 - `textColor` - default `#ffffff`
-- `position` - Vertical position: `top`, `bottom` or `center`
+- `position` - See **Positions**
+
+See also [Ken Burns parameters](#ken-burns-parameters)
 
 #### Layer type 'subtitle'
 - `fontPath` - See `defaults.layer.fontPath`
@@ -242,6 +283,21 @@ Loads a GLSL shader. See [gl.json5](https://github.com/mifi/editly/blob/master/e
 - `fragmentPath`
 - `vertexPath` (optional)
 
+### Position parameter
+
+Certain layers support the position parameter
+
+`position` can be one of either:
+  - `top`, `bottom` or `center` - vertical position (horizontally centered)
+  - An object `{ x, y, originX = 'left', originY = 'top' }`, where `{ x: 0, y: 0 }` is the upper left corner of the screen, and `{ x: 1, y: 1 }` is the lower right corner, `x` is relative to video width, `y` to height. `originX` and `originY` are optional, and specify the position's origin (anchor position) of the object.
+
+### Ken Burns parameters
+
+| Parameter  | Description | Default | |
+|-|-|-|-|
+| `zoomDirection` | Zoom direction for Ken Burns effect: `in`, `out` or `null` to disable | | |
+| `zoomAmount` | Zoom amount for Ken Burns effect | `0.1` | |
+
 ## Troubleshooting
 
 - If you get `Error: The specified module could not be found.`, try: `npm un -g editly && npm i -g --build-from-source editly` (see [#15](https://github.com/mifi/editly/issues/15))
@@ -256,9 +312,6 @@ Loads a GLSL shader. See [gl.json5](https://github.com/mifi/editly/blob/master/e
 - https://github.com/sjfricke/awesome-webgl
 - https://www.mltframework.org/docs/melt/
 
-## TODO
-
-- Keep source audio (See [#1](https://github.com/mifi/editly/issues/1))
 
 ---
 

audio.js

@@ -0,0 +1,139 @@
+const pMap = require('p-map');
+const { join, basename, resolve } = require('path');
+const execa = require('execa');
+const flatMap = require('lodash/flatMap');
+const fs = require('fs-extra');
+
+const { getFfmpegCommonArgs, getCutFromArgs, createConcatFile } = require('./ffmpeg');
+const { readFileStreams } = require('./util');
+
+module.exports = ({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose }) => {
+  async function editAudio({ clips, tmpDir }) {
+    if (clips.length === 0) return undefined;
+
+    console.log('Extracting audio or creating silence from all clips');
+
+    const mergedAudioPath = join(tmpDir, 'audio-merged.flac');
+
+    const segments = await pMap(clips, async (clip, i) => {
+      const clipAudioPath = join(tmpDir, `clip${i}-audio.flac`);
+
+      const audioLayers = clip.layers.filter(({ type, visibleFrom, visibleUntil }) => (
+        ['audio', 'video'].includes(type)
+        // TODO We don't support audio for visibleFrom/visibleUntil layers
+        && !visibleFrom && visibleUntil == null));
+
+      async function createSilence(outPath) {
+        if (verbose) console.log('create silence', clip.duration);
+        const args = [
+          '-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=44100',
+          '-sample_fmt', 's32',
+          '-ar', '48000',
+          '-t', clip.duration,
+          '-c:a', 'flac',
+          '-y',
+          outPath,
+        ];
+        await execa(ffmpegPath, args);
+      }
+
+      if (audioLayers.length > 0) {
+        const processedAudioLayersRaw = await pMap(audioLayers, async (audioLayer, j) => {
+          const { path, cutFrom, audioCutTo, framePtsFactor } = audioLayer;
+
+          const streams = await readFileStreams(ffprobePath, path);
+          if (!streams.some((s) => s.codec_type === 'audio')) return undefined;
+
+          const layerAudioPath = join(tmpDir, `clip${i}-layer${j}-audio.flac`);
+
+          try {
+            let atempoFilter;
+            if (Math.abs(framePtsFactor - 1) > 0.01) {
+              if (verbose) console.log('audio framePtsFactor', framePtsFactor);
+              const atempo = (1 / framePtsFactor);
+              if (!(atempo >= 0.5 && atempo <= 100)) { // Required range by ffmpeg
+                console.warn(`Audio speed ${atempo} is outside accepted range, using silence (clip ${i})`);
+                return undefined;
+              }
+              atempoFilter = `atempo=${atempo}`;
+            }
+
+            const cutToArg = (audioCutTo - cutFrom) * framePtsFactor;
+
+            const args = [
+              ...getFfmpegCommonArgs({ enableFfmpegLog }),
+              ...getCutFromArgs({ cutFrom }),
+              '-i', path,
+              '-t', cutToArg,
+              '-sample_fmt', 's32',
+              '-ar', '48000',
+              '-map', 'a:0', '-c:a', 'flac',
+              ...(atempoFilter ? ['-filter:a', atempoFilter] : []),
+              '-y',
+              layerAudioPath,
+            ];
+
+            // console.log(args);
+            await execa(ffmpegPath, args);
+          } catch (err) {
+            if (verbose) console.error('Cannot extract audio from video', path, err);
+            // Fall back to silence
+            await createSilence(layerAudioPath);
+          }
+
+          return { layerAudioPath, audioLayer };
+        }, { concurrency: 4 });
+
+        const processedAudioLayers = processedAudioLayersRaw.filter((p) => p);
+
+        if (processedAudioLayers.length > 1) {
+          // Merge/mix all layer's audio
+
+          const weights = processedAudioLayers.map(({ audioLayer }) => (audioLayer.mixVolume != null ? audioLayer.mixVolume : 1));
+          const args = [
+            ...getFfmpegCommonArgs({ enableFfmpegLog }),
+            ...flatMap(processedAudioLayers, ({ layerAudioPath }) => ['-i', layerAudioPath]),
+            '-filter_complex', `amix=inputs=${processedAudioLayers.length}:duration=longest:weights=${weights.join(' ')}`,
+            '-c:a', 'flac',
+            '-y',
+            clipAudioPath,
+          ];
+
+          await execa(ffmpegPath, args);
+        } else if (processedAudioLayers.length > 0) {
+          await fs.rename(processedAudioLayers[0].layerAudioPath, clipAudioPath);
+        } else {
+          await createSilence(clipAudioPath);
+        }
+      } else {
+        await createSilence(clipAudioPath);
+      }
+
+      // https://superuser.com/a/853262/658247
+      return resolve(clipAudioPath);
+    }, { concurrency: 4 });
+
+    const concatFilePath = join(tmpDir, 'audio-segments.txt');
+
+    console.log('Combining audio', segments.map((s) => basename(s)), concatFilePath);
+
+    await createConcatFile(segments, concatFilePath);
+
+    const args = [
+      ...getFfmpegCommonArgs({ enableFfmpegLog }),
+      '-f', 'concat', '-safe', '0',
+      '-i', concatFilePath,
+      '-c', 'flac',
+      '-y',
+      mergedAudioPath,
+    ];
+    await execa(ffmpegPath, args);
+
+    // TODO don't return audio if only silence?
+    return mergedAudioPath;
+  }
+
+  return {
+    editAudio,
+  };
+};

cli.js

@@ -32,6 +32,7 @@ const cli = meow(`
     --fps  FPS which all videos will be converted to
     --font-path  Set default font to a .ttf
     --audio-file-path  Add an audio track
+    --keep-source-audio  Keep audio from source files
 
     --fast, -f  Fast mode (low resolution and FPS, useful for getting a quick preview)
     --verbose, -v
@@ -45,6 +46,7 @@ const cli = meow(`
 `, {
   flags: {
     verbose: { type: 'boolean', alias: 'v' },
+    keepSourceAudio: { type: 'boolean' },
     fast: { type: 'boolean', alias: 'f' },
     transitionDuration: { type: 'number' },
     clipDuration: { type: 'number' },
@@ -92,7 +94,7 @@ const cli = meow(`
     params.clips = clips.map((clip) => ({ layers: [clip] }));
   }
 
-  const { verbose, transitionName, transitionDuration, clipDuration, width, height, fps, audioFilePath, fontPath, fast, out: outPath } = cli.flags;
+  const { verbose, transitionName, transitionDuration, clipDuration, width, height, fps, audioFilePath, fontPath, fast, out: outPath, keepSourceAudio } = cli.flags;
 
   if (transitionName || transitionDuration != null) {
     params.defaults.transition = {};
@@ -110,6 +112,7 @@ const cli = meow(`
 
   if (outPath) params.outPath = outPath;
   if (audioFilePath) params.audioFilePath = audioFilePath;
+  if (keepSourceAudio) params.keepSourceAudio = true;
   if (width) params.width = width;
   if (height) params.height = height;
   if (fps) params.fps = fps;

examples/README.md

@@ -1,24 +1,16 @@
 # Examples
 
-## Ken Burns zoom slideshow
+## Image slideshow with Ken Burns zoom
 
 ![](https://github.com/mifi/gifs/raw/master/kenburns.gif)
 
 [kenBurns.json5](https://github.com/mifi/editly/blob/master/examples/kenBurns.json5)
 
-```bash
-editly kenBurns.json5
-```
-
 ## News title
 
 ![](https://github.com/mifi/gifs/raw/master/newsTitle.gif)
 
-[kenBurns.json5](https://github.com/mifi/editly/blob/master/examples/newsTitle.json5)
-
-```bash
-editly newsTitle.json5
-```
+[newsTitle.json5](https://github.com/mifi/editly/blob/master/examples/newsTitle.json5)
 
 ## Resize modes
 
@@ -26,29 +18,37 @@ editly newsTitle.json5
 
 [resizeHorizontal.json5](https://github.com/mifi/editly/blob/master/examples/resizeHorizontal.json5)
 
-```bash
-editly resizeHorizontal.json5
-```
-
 ## Speed up / slow down with cutting
 
 ![](https://github.com/mifi/gifs/raw/master/speedTest.gif)
 
 [speedTest.json5](https://github.com/mifi/editly/blob/master/examples/speedTest.json5)
 
-```bash
-editly speedTest.json5
-```
-
 ## Title and subtitle
 
 ![](https://github.com/mifi/gifs/raw/master/subtitle.gif)
 
 [subtitle.json5](https://github.com/mifi/editly/blob/master/examples/subtitle.json5)
 
-```bash
-editly subtitle.json5
-```
+## Video overlays with alpha channel
+
+[alpha.json5](https://github.com/mifi/editly/blob/master/examples/alpha.json5)
+
+## Image overlays with alpha channel
+
+![](https://github.com/mifi/gifs/raw/master/imageOverlay.gif)
+
+[imageOverlay.json5](https://github.com/mifi/editly/blob/master/examples/imageOverlay.json5)
+
+## Partial overlays (B-roll)
+
+[visibleFromUntil.json5](https://github.com/mifi/editly/blob/master/examples/visibleFromUntil.json5)
+
+## Audio layers
+
+- [audio1.json5](https://github.com/mifi/editly/blob/master/examples/audio1.json5)
+- [audio2.json5](https://github.com/mifi/editly/blob/master/examples/audio2.json5)
+
 
 ## Custom HTML5 canvas Javascript
 
@@ -56,7 +56,6 @@ editly subtitle.json5
 
 [customCanvas.js](https://github.com/mifi/editly/blob/master/examples/customCanvas.js)
 
-
 ```bash
 node customCanvas.js
 ```
@@ -67,11 +66,11 @@ node customCanvas.js
 
 [customFabric.js](https://github.com/mifi/editly/blob/master/examples/customFabric.js)
 
-
 ```bash
 node customFabric.js
 ```
 
+
 ## LosslessCut tutorial
 
 [This video](https://www.youtube.com/watch?v=pYHMxXy05Jg) was created with [losslesscut.json5](https://github.com/mifi/editly/blob/master/examples/losslesscut.json5)

examples/alpha.json5

@@ -0,0 +1,14 @@
+{
+  // enableFfmpegLog: true,
+  outPath: './alpha.mp4',
+  clips: [
+    { duration: 2, layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 },
+      { type: 'video', path: './assets/dancer1.webm', cutFrom: 0, cutTo: 6 },
+    ] },
+    { layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 },
+      { type: 'video', path: './assets/dancer1.webm' },
+    ] },
+  ],
+}

examples/audio1.json5

@@ -0,0 +1,25 @@
+{
+  // enableFfmpegLog: true,
+  outPath: './audio1.mp4',
+  defaults: {
+    transition: null,
+  },
+  clips: [
+    { duration: 0.5, layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }] },
+
+    { layers: [
+      { type: 'fill-color' },
+      { type: 'title', text: 'test' },
+      { type: 'audio', path: './assets/High [NCS Release] - JPB  (No Copyright Music)-R8ZRCXy5vhA.m4a', cutFrom: 2, cutTo: 5 }] },
+
+    { layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 12, cutTo: 14, mixVolume: 0 },
+      { type: 'audio', path: './assets/High [NCS Release] - JPB  (No Copyright Music)-R8ZRCXy5vhA.m4a', mixVolume: 0.1 }] },
+
+    { duration: 2, layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 },
+      { type: 'audio', path: './assets/High [NCS Release] - JPB  (No Copyright Music)-R8ZRCXy5vhA.m4a', cutFrom: 2, cutTo: 3, mixVolume: 0.5 }] },
+
+    { duration: 1.8, layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 10, cutTo: 11 }] },
+  ],
+}

examples/audio2.json5

@@ -0,0 +1,21 @@
+{
+  // enableFfmpegLog: true,
+  outPath: './audio2.mp4',
+  clips: [
+    { duration: 0.5, layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }] },
+
+    { layers: [
+      { type: 'title', text: 'test' },
+      { type: 'audio', path: './assets/High [NCS Release] - JPB  (No Copyright Music)-R8ZRCXy5vhA.m4a' }] },
+
+    { layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 12, cutTo: 14, mixVolume: 0.7 },
+      { type: 'audio', path: './assets/High [NCS Release] - JPB  (No Copyright Music)-R8ZRCXy5vhA.m4a', mixVolume: 0.3 }] },
+
+    { layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 },
+      { type: 'audio', path: './assets/High [NCS Release] - JPB  (No Copyright Music)-R8ZRCXy5vhA.m4a' }] },
+
+    { layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 10, cutTo: 11 }] },
+  ],
+}

examples/customFabric.js

@@ -2,8 +2,8 @@ const editly = require('..');
 
 /* eslint-disable spaced-comment,no-param-reassign */
 
-async function func({ width, height, fabric, canvas }) {
-  async function onRender(progress) {
+async function func({ width, height, fabric }) {
+  async function onRender(progress, canvas) {
     canvas.backgroundColor = 'hsl(33, 100%, 50%)';
 
     const text = new fabric.Text(`PROGRESS\n${Math.floor(progress * 100)}%`, {

examples/imageOverlay.json5

@@ -0,0 +1,12 @@
+{
+  outPath: './imageOverlay.mp4',
+  clips: [
+    { layers: [
+      { type: 'video', path: './assets/IMG_4605.MOV', cutTo: 2 },
+      { type: 'image-overlay', path: './assets/overlay.svg', width: 0.2, position: { x: 0.95, y: 0.03, originX: 'right' } },
+      { type: 'image-overlay', path: './assets/emoji.png', visibleUntil: 0.5, zoomDirection: 'in' },
+      { type: 'image-overlay', path: './assets/emoji2.svg', position: 'top', visibleFrom: 0.7, visibleUntil: 1.5, width: 0.2 },
+      { type: 'image-overlay', path: './assets/emoji2.svg', position: 'bottom', visibleFrom: 0.7, visibleUntil: 1.5, height: 0.2 },
+    ] },
+  ],
+}

examples/smartFit.json5

@@ -0,0 +1,13 @@
+{
+  // enableFfmpegLog: true,
+  outPath: './smartFit.mp4',
+  defaults: {
+    transition: null,
+    layer: { backgroundColor: 'white' },
+  },
+  clips: [
+    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2 }] },
+    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'contain' }] },
+    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'stretch' }] },
+  ],
+}

examples/subtitle.json5

@@ -2,6 +2,7 @@
   outPath: './subtitle.mp4',
   defaults: {
     layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' },
+    layerType: { 'fill-color': { color: '#00aa00' } }
   },
   clips: [
     { duration: 2, layers: [
@@ -9,5 +10,9 @@
       { type: 'subtitle', text: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident.' },
       { type: 'title', position: 'top', text: 'Subtitles' },
     ] },
+    { duration: 2, layers: [
+      { type: 'fill-color' },
+      { type: 'title', position: { x: 0, y: 1, originY: 'bottom' }, text: 'Custom position', zoomDirection: null },
+    ] },
   ],
 }

examples/timeoutTest.json5

@@ -0,0 +1,7 @@
+{
+  outPath: './timeoutTest.mp4',
+  clips: [
+    { duration: 1.5, transition: { name: 'crosszoom', duration: 0.3 }, layers: [{ type: 'video', path: './assets/DJI_0156.mov', cutTo: 58 }] },
+    { duration: 3, transition: { name: 'fade' }, layers: [{ type: 'video', path: './assets/DJI_0156.mov', cutFrom: 0 }] },
+  ],
+}

examples/visibleFromUntil.json5

@@ -0,0 +1,18 @@
+{
+  // enableFfmpegLog: true,
+  outPath: './visibleFromUntil.mp4',
+  clips: [
+    { duration: 2, layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 },
+      { type: 'video', path: './assets/dancer1.webm', cutFrom: 0, cutTo: 6, visibleFrom: 0.5, visibleUntil: 1 },
+    ] },
+    { duration: 2, layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 7.5, cutTo: 10.5 },
+      { type: 'news-title', text: 'Hei', visibleFrom: 0.5, visibleUntil: 1  },
+    ] },
+    { layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 14, cutTo: 18 },
+      { type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 1, visibleFrom: 1, visibleUntil: 2 },
+    ] },
+  ],
+}

ffmpeg.js

@@ -0,0 +1,19 @@
+const fs = require('fs-extra');
+
+const getFfmpegCommonArgs = ({ enableFfmpegLog }) => (enableFfmpegLog ? [] : ['-hide_banner', '-loglevel', 'error']);
+
+const getCutFromArgs = ({ cutFrom }) => (cutFrom ? ['-ss', cutFrom] : []);
+
+const getCutToArgs = ({ cutTo, cutFrom, framePtsFactor }) => (cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []);
+
+async function createConcatFile(segments, concatFilePath) {
+  // https://superuser.com/questions/787064/filename-quoting-in-ffmpeg-concat
+  await fs.writeFile(concatFilePath, segments.map((seg) => `file '${seg.replace(/'/g, "'\\''")}'`).join('\n'));
+}
+
+module.exports = {
+  getFfmpegCommonArgs,
+  getCutFromArgs,
+  getCutToArgs,
+  createConcatFile,
+};

index.js

@@ -1,17 +1,18 @@
 const execa = require('execa');
 const assert = require('assert');
 const pMap = require('p-map');
-const { basename, join } = require('path');
+const { basename, join, dirname } = require('path');
 const flatMap = require('lodash/flatMap');
 const JSON5 = require('json5');
 const fs = require('fs-extra');
 
-const { parseFps, readFileInfo, multipleOf2 } = require('./util');
+const { parseFps, readVideoFileInfo, readAudioFileInfo, multipleOf2 } = require('./util');
 const { registerFont } = require('./sources/fabricFrameSource');
 const { createFrameSource } = require('./sources/frameSource');
 const { calcTransition } = require('./transitions');
 
 const GlTransitions = require('./glTransitions');
+const Audio = require('./audio');
 
 // Cache
 const loadedFonts = [];
@@ -21,7 +22,6 @@ const checkTransition = (transition) => assert(transition == null || typeof tran
 
 const assertFileExists = async (path, enableRemote) => assert((enableRemote && path.startsWith('http')) || await fs.exists(path), `File does not exist ${path}`);
 
-
 module.exports = async (config = {}) => {
   const {
     // Testing options:
@@ -36,6 +36,7 @@ module.exports = async (config = {}) => {
     fps: requestedFps,
     defaults: defaultsIn = {},
     audioFilePath: audioFilePathIn,
+    keepSourceAudio,
 
     ffmpegPath = 'ffmpeg',
     ffprobePath = 'ffprobe',
@@ -50,7 +51,8 @@ module.exports = async (config = {}) => {
 
   const isGif = outPath.toLowerCase().endsWith('.gif');
 
-  const audioFilePath = isGif ? undefined : audioFilePathIn;
+  let audioFilePath;
+  if (!isGif) audioFilePath = audioFilePathIn;
 
   if (audioFilePath) await assertFileExists(audioFilePath, true);
 
@@ -74,8 +76,7 @@ module.exports = async (config = {}) => {
   async function handleLayer(layer) {
     const { type, ...restLayer } = layer;
 
-    // https://github.com/mifi/editly/issues/39
-    if (type === 'image') {
+    if (['image', 'image-overlay'].includes(type)) {
       await assertFileExists(restLayer.path, true);
     } else if (type === 'gl') {
       await assertFileExists(restLayer.fragmentPath);
@@ -83,7 +84,7 @@ module.exports = async (config = {}) => {
 
     if (['fabric', 'canvas'].includes(type)) assert(typeof layer.func === 'function', '"func" must be a function');
 
-    if (['image', 'fabric', 'canvas', 'gl', 'radial-gradient', 'linear-gradient', 'fill-color'].includes(type)) return layer;
+    if (['image', 'image-overlay', 'fabric', 'canvas', 'gl', 'radial-gradient', 'linear-gradient', 'fill-color'].includes(type)) return layer;
 
     // TODO if random-background radial-gradient linear etc
     if (type === 'pause') return handleLayer({ ...restLayer, type: 'fill-color' });
@@ -134,66 +135,134 @@ module.exports = async (config = {}) => {
   }
 
   const clips = await pMap(clipsIn, async (clip, clipIndex) => {
-    const { transition: userTransition, duration: userDuration, layers } = clip;
+    const { transition: userTransition, duration: userClipDuration, layers } = clip;
 
     checkTransition(userTransition);
 
     const videoLayers = layers.filter((layer) => layer.type === 'video');
-    assert(videoLayers.length <= 1, 'Max 1 video per layer');
 
-    const userOrDefaultDuration = userDuration || defaults.duration;
-    if (videoLayers.length === 0) assert(userOrDefaultDuration, `Duration is required for clip ${clipIndex}`);
+    const userClipDurationOrDefault = userClipDuration || defaults.duration;
+    if (videoLayers.length === 0) assert(userClipDurationOrDefault, `Duration parameter is required for videoless clip ${clipIndex}`);
 
-    let duration = userOrDefaultDuration;
+    const transition = calcTransition(defaults, userTransition, clipIndex === clipsIn.length - 1);
 
-    const layersOut = flatMap(await pMap(layers, async (layerIn) => {
-      const layer = { ...defaults.layer, ...layerIn };
-      const { type } = layer;
+    let layersOut = flatMap(await pMap(layers, async (layerIn) => {
+      const globalLayerDefaults = defaults.layer || {};
+      const thisLayerDefaults = (defaults.layerType || {})[layerIn.type];
+      const layer = { ...globalLayerDefaults, ...thisLayerDefaults, ...layerIn };
+      const { type, path } = layer;
 
       if (type === 'video') {
-        const { cutFrom: cutFromIn, cutTo: cutToIn, path } = layer;
-        const fileInfo = await readFileInfo(ffprobePath, path);
-        const { duration: fileDuration, width: widthIn, height: heightIn, framerateStr, rotation } = fileInfo;
-        let cutFrom;
-        let cutTo;
-        let trimmedSourceDuration = fileDuration;
-        if (cutFromIn != null || cutToIn != null) {
-          cutFrom = Math.min(Math.max(0, cutFromIn || 0), fileDuration);
-          cutTo = Math.min(Math.max(cutFrom, cutToIn || fileDuration), fileDuration);
-          assert(cutFrom < cutTo, 'cutFrom must be lower than cutTo');
-
-          trimmedSourceDuration = cutTo - cutFrom;
-        }
+        const { duration: fileDuration, width: widthIn, height: heightIn, framerateStr, rotation } = await readVideoFileInfo(ffprobePath, path);
+        let { cutFrom, cutTo } = layer;
+        if (!cutFrom) cutFrom = 0;
+        cutFrom = Math.max(cutFrom, 0);
+        cutFrom = Math.min(cutFrom, fileDuration);
 
-        // If user specified duration, means that should be the output duration
-        let framePtsFactor;
-        if (userDuration) {
-          duration = userDuration;
-          framePtsFactor = userDuration / trimmedSourceDuration;
-        } else {
-          duration = trimmedSourceDuration;
-          framePtsFactor = 1;
-        }
+        if (!cutTo) cutTo = fileDuration;
+        cutTo = Math.max(cutTo, cutFrom);
+        cutTo = Math.min(cutTo, fileDuration);
+        assert(cutFrom < cutTo, 'cutFrom must be lower than cutTo');
+
+        const inputDuration = cutTo - cutFrom;
 
         const isRotated = rotation === 90 || rotation === 270;
         const width = isRotated ? heightIn : widthIn;
         const height = isRotated ? widthIn : heightIn;
 
-        return { ...layer, cutFrom, cutTo, width, height, framerateStr, framePtsFactor };
+        // Compensate for transition duration
+        const audioCutTo = Math.max(cutFrom, cutTo - transition.duration);
+
+        return { ...layer, cutFrom, cutTo, audioCutTo, inputDuration, width, height, framerateStr };
       }
 
+      // Audio is handled later
+      if (type === 'audio') return layer;
+
       return handleLayer(layer);
     }, { concurrency: 1 }));
 
-    const transition = calcTransition(defaults, userTransition);
+    let clipDuration = userClipDurationOrDefault;
+
+    const firstVideoLayer = layersOut.find((layer) => layer.type === 'video');
+    if (firstVideoLayer && !userClipDuration) clipDuration = firstVideoLayer.inputDuration;
+    assert(clipDuration);
+
+    // We need to map again, because for audio, we need to know the correct clipDuration
+    layersOut = await pMap(layersOut, async (layerIn) => {
+      const { type, path, visibleUntil, visibleFrom = 0 } = layerIn;
+
+      // This feature allows the user to show another layer overlayed (or replacing) parts of the lower layers (visibleFrom - visibleUntil)
+      const visibleDuration = ((visibleUntil || clipDuration) - visibleFrom);
+      assert(visibleDuration > 0 && visibleDuration <= clipDuration, `Invalid visibleFrom ${visibleFrom} or visibleUntil ${visibleUntil}`);
+      // TODO Also need to handle video layers (framePtsFactor etc)
+      // TODO handle audio in case of visibleFrom/visibleTo
+
+      const layer = { ...layerIn, visibleFrom, visibleDuration };
+
+      if (type === 'audio') {
+        const { duration: fileDuration } = await readAudioFileInfo(ffprobePath, path);
+        let { cutFrom, cutTo } = layer;
+
+        // console.log({ cutFrom, cutTo, fileDuration, clipDuration });
+
+        if (!cutFrom) cutFrom = 0;
+        cutFrom = Math.max(cutFrom, 0);
+        cutFrom = Math.min(cutFrom, fileDuration);
+
+        if (!cutTo) cutTo = cutFrom + clipDuration;
+        cutTo = Math.max(cutTo, cutFrom);
+        cutTo = Math.min(cutTo, fileDuration);
+        assert(cutFrom < cutTo, 'cutFrom must be lower than cutTo');
+
+        const inputDuration = cutTo - cutFrom;
+
+        const framePtsFactor = clipDuration / inputDuration;
+
+        // Compensate for transition duration
+        const audioCutTo = Math.max(cutFrom, cutTo - transition.duration);
+
+        return { ...layer, cutFrom, cutTo, audioCutTo, framePtsFactor };
+      }
+
+      if (layer.type === 'video') {
+        const { inputDuration } = layer;
+
+        let framePtsFactor;
+
+        // If user explicitly specified duration for clip, it means that should be the output duration of the video
+        if (userClipDuration) {
+          // Later we will speed up or slow down video using this factor
+          framePtsFactor = userClipDuration / inputDuration;
+        } else {
+          framePtsFactor = 1;
+        }
+
+        return { ...layer, framePtsFactor };
+      }
+
+      return layer;
+    });
 
     return {
       transition,
-      duration,
+      duration: clipDuration,
       layers: layersOut,
     };
   }, { concurrency: 1 });
 
+  const { editAudio } = Audio({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose });
+
+  const outDir = dirname(outPath);
+  const tmpDir = join(outDir, 'editly-tmp');
+  if (verbose) console.log({ tmpDir });
+  await fs.remove(tmpDir);
+  await fs.mkdirp(tmpDir);
+
+  if (!audioFilePath && keepSourceAudio) {
+    audioFilePath = await editAudio({ clips, tmpDir });
+  }
+
   if (verbose) console.log(JSON5.stringify(clips, null, 2));
 
   // Try to detect parameters from first video
@@ -329,110 +398,130 @@ module.exports = async (config = {}) => {
   let frameSource1;
   let frameSource2;
 
+  let frameSource1Data;
+
+  let totalFramesWritten = 0;
+  let fromClipFrameAt = 0;
+  let toClipFrameAt = 0;
+
+  let transitionFromClipId = 0;
+
+  const getTransitionToClipId = () => transitionFromClipId + 1;
+  const getTransitionFromClip = () => clips[transitionFromClipId];
+  const getTransitionToClip = () => clips[getTransitionToClipId()];
+
+  const getSource = async (clip, clipIndex) => createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, framerateStr });
+  const getTransitionFromSource = async () => getSource(getTransitionFromClip(), transitionFromClipId);
+  const getTransitionToSource = async () => (getTransitionToClip() && getSource(getTransitionToClip(), getTransitionToClipId()));
+
   try {
     outProcess = startFfmpegWriterProcess();
     if (onProcessStart) onProcessStart(outProcess);
     let outProcessError;
 
-    // If we don't catch it here, the whole process will crash and we cannot process the error
+    // If we don't handle it here, the whole Node process will crash and we cannot process the error
     outProcess.stdin.on('error', (err) => {
       console.error('Output ffmpeg caught error', err);
       outProcessError = err;
     });
 
-    let totalFrameCount = 0;
-    let fromClipFrameCount = 0;
-    let toClipFrameCount = 0;
-
-    let transitionFromClipId = 0;
-
-    const getTransitionToClipId = () => transitionFromClipId + 1;
-    const getTransitionFromClip = () => clips[transitionFromClipId];
-    const getTransitionToClip = () => clips[getTransitionToClipId()];
-
-    const getSource = (clip, clipIndex) => createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, enableFfmpegLog, framerateStr });
-
-    const getTransitionToSource = async () => (getTransitionToClip() && getSource(getTransitionToClip(), getTransitionToClipId()));
-    frameSource1 = await getSource(getTransitionFromClip(), transitionFromClipId);
+    frameSource1 = await getTransitionFromSource();
     frameSource2 = await getTransitionToSource();
 
     // eslint-disable-next-line no-constant-condition
     while (true) {
-      const fromClipNumFrames = Math.round(getTransitionFromClip().duration * fps);
-      const toClipNumFrames = getTransitionToClip() && Math.round(getTransitionToClip().duration * fps);
-      const fromClipProgress = fromClipFrameCount / fromClipNumFrames;
-      const toClipProgress = getTransitionToClip() && toClipFrameCount / toClipNumFrames;
-      const frameData1 = await frameSource1.readNextFrame(fromClipProgress);
+      const transitionToClip = getTransitionToClip();
+      const transitionFromClip = getTransitionFromClip();
+      const fromClipNumFrames = Math.round(transitionFromClip.duration * fps);
+      const toClipNumFrames = transitionToClip && Math.round(transitionToClip.duration * fps);
+      const fromClipProgress = fromClipFrameAt / fromClipNumFrames;
+      const toClipProgress = transitionToClip && toClipFrameAt / toClipNumFrames;
+      const fromClipTime = transitionFromClip.duration * fromClipProgress;
+      const toClipTime = transitionToClip && transitionToClip.duration * toClipProgress;
 
-      const clipTransition = getTransitionFromClip().transition;
+      const currentTransition = transitionFromClip.transition;
 
-      const transitionNumFrames = Math.round(clipTransition.duration * fps);
+      const transitionNumFrames = Math.round(currentTransition.duration * fps);
 
       // Each clip has two transitions, make sure we leave enough room:
       const transitionNumFramesSafe = Math.floor(Math.min(Math.min(fromClipNumFrames, toClipNumFrames != null ? toClipNumFrames : Number.MAX_SAFE_INTEGER) / 2, transitionNumFrames));
       // How many frames into the transition are we? negative means not yet started
-      const transitionFrameAt = fromClipFrameCount - (fromClipNumFrames - transitionNumFramesSafe);
-
-      if (verbose) console.log('Frame', totalFrameCount, 'from', fromClipFrameCount, `(clip ${transitionFromClipId})`, 'to', toClipFrameCount, `(clip ${getTransitionToClipId()})`);
+      const transitionFrameAt = fromClipFrameAt - (fromClipNumFrames - transitionNumFramesSafe);
 
       if (!verbose) {
-        const percentDone = Math.floor(100 * (totalFrameCount / estimatedTotalFrames));
-        if (totalFrameCount % 10 === 0) process.stdout.write(`${String(percentDone).padStart(3, ' ')}% `);
+        const percentDone = Math.floor(100 * (totalFramesWritten / estimatedTotalFrames));
+        if (totalFramesWritten % 10 === 0) process.stdout.write(`${String(percentDone).padStart(3, ' ')}% `);
       }
 
-      if (!frameData1 || transitionFrameAt >= transitionNumFramesSafe - 1) {
-      // if (!frameData1 || transitionFrameAt >= transitionNumFramesSafe) {
-        console.log('Done with transition, switching to next clip');
+      // console.log({ transitionFrameAt, transitionNumFramesSafe })
+      // const transitionLastFrameIndex = transitionNumFramesSafe - 1;
+      const transitionLastFrameIndex = transitionNumFramesSafe;
+
+      // Done with transition?
+      if (transitionFrameAt >= transitionLastFrameIndex) {
         transitionFromClipId += 1;
+        console.log(`Done with transition, switching to next transitionFromClip (${transitionFromClipId})`);
 
         if (!getTransitionFromClip()) {
           console.log('No more transitionFromClip, done');
           break;
         }
 
-        // Cleanup old, swap and load next
+        // Cleanup completed frameSource1, swap and load next frameSource2
         await frameSource1.close();
         frameSource1 = frameSource2;
         frameSource2 = await getTransitionToSource();
 
-        fromClipFrameCount = transitionNumFramesSafe;
-        toClipFrameCount = 0;
-      } else {
-        let outFrameData;
-        if (frameSource2 && transitionFrameAt >= 0) {
-          if (verbose) console.log('Transition', 'frame', transitionFrameAt, '/', transitionNumFramesSafe, clipTransition.name, `${clipTransition.duration}s`);
-
-          const frameData2 = await frameSource2.readNextFrame(toClipProgress);
-          toClipFrameCount += 1;
-
-          if (frameData2) {
-            const progress = transitionFrameAt / transitionNumFramesSafe;
-            const easedProgress = clipTransition.easingFunction(progress);
-
-            if (verbose) console.time('runTransitionOnFrame');
-            outFrameData = runTransitionOnFrame({ fromFrame: frameData1, toFrame: frameData2, progress: easedProgress, transitionName: clipTransition.name, transitionParams: clipTransition.params });
-            if (verbose) console.timeEnd('runTransitionOnFrame');
-          } else {
-            console.warn('Got no frame data from clip 2!');
-            // We have reached end of clip2 but transition is not complete
-            // Pass thru
-            // TODO improve, maybe cut it short
-            outFrameData = frameData1;
-          }
+        fromClipFrameAt = transitionLastFrameIndex;
+        toClipFrameAt = 0;
+
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      const newFrameSource1Data = await frameSource1.readNextFrame({ time: fromClipTime });
+      // If we got no data, use the old data
+      // TODO maybe abort?
+      if (newFrameSource1Data) frameSource1Data = newFrameSource1Data;
+      else console.warn('No frame data returned, using last frame');
+
+      const isInTransition = frameSource2 && transitionNumFramesSafe > 0 && transitionFrameAt >= 0;
+
+      let outFrameData;
+      if (isInTransition) {
+        const frameSource2Data = await frameSource2.readNextFrame({ time: toClipTime });
+
+        if (frameSource2Data) {
+          const progress = transitionFrameAt / transitionNumFramesSafe;
+          const easedProgress = currentTransition.easingFunction(progress);
+
+          // if (verbose) console.time('runTransitionOnFrame');
+          outFrameData = runTransitionOnFrame({ fromFrame: frameSource1Data, toFrame: frameSource2Data, progress: easedProgress, transitionName: currentTransition.name, transitionParams: currentTransition.params });
+          // if (verbose) console.timeEnd('runTransitionOnFrame');
         } else {
-          outFrameData = frameData1;
+          console.warn('Got no frame data from transitionToClip!');
+          // We have probably reached end of clip2 but transition is not complete. Just pass thru clip1
+          outFrameData = frameSource1Data;
         }
+      } else {
+        // Not in transition. Pass thru clip 1
+        outFrameData = frameSource1Data;
+      }
 
-        // If we don't await we get EINVAL when dealing with high resolution files (big writes)
-        await new Promise((r) => outProcess.stdin.write(outFrameData, () => r()));
+      if (verbose) {
+        if (isInTransition) console.log('Writing frame:', totalFramesWritten, 'from clip', transitionFromClipId, `(frame ${fromClipFrameAt})`, 'to clip', getTransitionToClipId(), `(frame ${toClipFrameAt} / ${transitionNumFramesSafe})`, currentTransition.name, `${currentTransition.duration}s`);
+        else console.log('Writing frame:', totalFramesWritten, 'from clip', transitionFromClipId, `(frame ${fromClipFrameAt})`);
+      }
 
-        if (outProcessError) throw outProcessError;
+      // If we don't wait for callback, then we get EINVAL when dealing with high resolution files (big writes)
+      await new Promise((r) => outProcess.stdin.write(outFrameData, () => r()));
 
-        fromClipFrameCount += 1;
-      }
+      if (outProcessError) throw outProcessError;
 
-      totalFrameCount += 1;
-    }
+      totalFramesWritten += 1;
+      fromClipFrameAt += 1;
+      if (isInTransition) toClipFrameAt += 1;
+    } // End while loop
 
     outProcess.stdin.end();
 
@@ -440,12 +529,11 @@ module.exports = async (config = {}) => {
     console.log(outPath);
   } catch (err) {
     console.error('Loop failed', err);
-    if (outProcess) {
-      outProcess.kill();
-    }
+    if (outProcess) outProcess.kill();
   } finally {
     if (frameSource1) await frameSource1.close();
     if (frameSource2) await frameSource2.close();
+    await fs.remove(tmpDir);
   }
 
   try {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "editly",
   "description": "Simple, sexy, declarative video editing",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "main": "index.js",
   "author": "Mikael Finstad <finstaden@gmail.com>",
   "license": "MIT",

sources/fabricFrameSource.js

@@ -7,6 +7,7 @@ const { createCanvas } = nodeCanvas;
 const { canvasToRgba } = require('./shared');
 const { getRandomGradient, getRandomColors } = require('../colors');
 const { easeOutExpo } = require('../transitions');
+const { getPositionProps } = require('../util');
 
 // http://fabricjs.com/kitchensink
 
@@ -56,10 +57,21 @@ async function createFabricFrameSource(func, { width, height, ...rest }) {
   };
 }
 
+const loadImage = async (path) => new Promise((resolve) => fabric.util.loadImage(fileUrl(path), resolve));
+
+function getZoomParams({ progress, zoomDirection, zoomAmount }) {
+  let scaleFactor = 1;
+  if (zoomDirection === 'in') scaleFactor = (1 + zoomAmount * progress);
+  else if (zoomDirection === 'out') scaleFactor = (1 + zoomAmount * (1 - progress));
+  return scaleFactor;
+}
+
 async function imageFrameSource({ verbose, params, width, height }) {
-  if (verbose) console.log('Loading', params.path);
+  const { path, zoomDirection = 'in', zoomAmount = 0.1 } = params;
 
-  const imgData = await new Promise((resolve) => fabric.util.loadImage(fileUrl(params.path), resolve));
+  if (verbose) console.log('Loading', path);
+
+  const imgData = await loadImage(path);
 
   const getImg = () => new fabric.Image(imgData, {
     originX: 'center',
@@ -76,15 +88,10 @@ async function imageFrameSource({ verbose, params, width, height }) {
   if (blurredImg.height > blurredImg.width) blurredImg.scaleToWidth(width);
   else blurredImg.scaleToHeight(height);
 
-
   async function onRender(progress, canvas) {
-    const { zoomDirection = 'in', zoomAmount = 0.1 } = params;
-
     const img = getImg();
 
-    let scaleFactor = 1;
-    if (zoomDirection === 'in') scaleFactor = (1 + progress * zoomAmount);
-    else if (zoomDirection === 'out') scaleFactor = (1 + zoomAmount * (1 - progress));
+    const scaleFactor = getZoomParams({ progress, zoomDirection, zoomAmount });
 
     if (img.height > img.width) img.scaleToHeight(height * scaleFactor);
     else img.scaleToWidth(width * scaleFactor);
@@ -232,8 +239,40 @@ async function subtitleFrameSource({ width, height, params }) {
   return { onRender };
 }
 
+async function imageOverlayFrameSource({ params, width, height }) {
+  const { path, position, width: relWidth, height: relHeight, zoomDirection, zoomAmount = 0.1 } = params;
+
+  const imgData = await loadImage(path);
+
+  const { left, top, originX, originY } = getPositionProps({ position, width, height });
+
+  const img = new fabric.Image(imgData, {
+    originX,
+    originY,
+    left,
+    top,
+  });
+
+  async function onRender(progress, canvas) {
+    const scaleFactor = getZoomParams({ progress, zoomDirection, zoomAmount });
+
+    if (relWidth != null) {
+      img.scaleToWidth(relWidth * width * scaleFactor);
+    } else if (relHeight != null) {
+      img.scaleToHeight(relHeight * height * scaleFactor);
+    } else {
+      // Default to screen width
+      img.scaleToWidth(width * scaleFactor);
+    }
+
+    canvas.add(img);
+  }
+
+  return { onRender };
+}
+
 async function titleFrameSource({ width, height, params }) {
-  const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center' } = params;
+  const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center', zoomDirection = 'in', zoomAmount = 0.2 } = params;
 
   async function onRender(progress, canvas) {
     // console.log('progress', progress);
@@ -242,7 +281,7 @@ async function titleFrameSource({ width, height, params }) {
 
     const fontSize = Math.round(min * 0.1);
 
-    const scale = (1 + progress * 0.2).toFixed(4);
+    const scaleFactor = getZoomParams({ progress, zoomDirection, zoomAmount });
 
     const textBox = new fabric.Textbox(text, {
       fill: textColor,
@@ -252,25 +291,18 @@ async function titleFrameSource({ width, height, params }) {
       width: width * 0.8,
     });
 
+    // We need the text as an image in order to scale it
     const textImage = await new Promise((r) => textBox.cloneAsImage(r));
 
-    let originY = 'center';
-    let top = height / 2;
-    if (position === 'top') {
-      originY = 'top';
-      top = height * 0.05;
-    } else if (position === 'bottom') {
-      originY = 'bottom';
-      top = height;
-    }
+    const { left, top, originX, originY } = getPositionProps({ position, width, height });
 
     textImage.set({
-      originX: 'center',
+      originX,
       originY,
-      left: width / 2,
+      left,
       top,
-      scaleX: scale,
-      scaleY: scale,
+      scaleX: scaleFactor,
+      scaleY: scaleFactor,
     });
     canvas.add(textImage);
   }
@@ -363,6 +395,7 @@ module.exports = {
   radialGradientFrameSource,
   linearGradientFrameSource,
   imageFrameSource,
+  imageOverlayFrameSource,
 
   createFabricCanvas,
   renderFabricCanvas,

sources/frameSource.js

@@ -1,15 +1,16 @@
 const assert = require('assert');
 const pMap = require('p-map');
 
-const { rgbaToFabricImage, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource, newsTitleFrameSource, createFabricCanvas, renderFabricCanvas } = require('./fabricFrameSource');
+const { rgbaToFabricImage, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, imageOverlayFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource, newsTitleFrameSource, createFabricCanvas, renderFabricCanvas } = require('./fabricFrameSource');
 const createVideoFrameSource = require('./videoFrameSource');
 const { createGlFrameSource } = require('./glFrameSource');
 
-
-async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, enableFfmpegLog, framerateStr }) {
+async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, framerateStr }) {
   const { layers, duration } = clip;
 
-  const layerFrameSources = await pMap(layers, async (layer, layerIndex) => {
+  const visualLayers = layers.filter((layer) => layer.type !== 'audio');
+
+  const layerFrameSources = await pMap(visualLayers, async (layer, layerIndex) => {
     const { type, ...params } = layer;
     console.log('createFrameSource', type, 'clip', clipIndex, 'layer', layerIndex);
 
@@ -19,6 +20,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
       canvas: createCustomCanvasFrameSource,
       fabric: async (opts) => createFabricFrameSource(customFabricFrameSource, opts),
       image: async (opts) => createFabricFrameSource(imageFrameSource, opts),
+      'image-overlay': async (opts) => createFabricFrameSource(imageOverlayFrameSource, opts),
       title: async (opts) => createFabricFrameSource(titleFrameSource, opts),
       subtitle: async (opts) => createFabricFrameSource(subtitleFrameSource, opts),
       'linear-gradient': async (opts) => createFabricFrameSource(linearGradientFrameSource, opts),
@@ -29,25 +31,33 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
     const createFrameSourceFunc = frameSourceFuncs[type];
     assert(createFrameSourceFunc, `Invalid type ${type}`);
 
-    return createFrameSourceFunc({ ffmpegPath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
+    const frameSource = await createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
+    return { layer, frameSource };
   }, { concurrency: 1 });
 
-  async function readNextFrame(progress) {
+  async function readNextFrame({ time }) {
     const canvas = createFabricCanvas({ width, height });
 
     // eslint-disable-next-line no-restricted-syntax
-    for (const frameSource of layerFrameSources) {
-      const rgba = await frameSource.readNextFrame(progress, canvas);
-      // Frame sources can either render to the provided canvas and return nothing
-      // OR return an raw RGBA blob which will be drawn onto the canvas
-      if (rgba) {
-        // Optimization: Don't need to draw to canvas if there's only one layer
-        if (layerFrameSources.length === 1) return rgba;
+    for (const { frameSource, layer } of layerFrameSources) {
+      // console.log({ visibleFrom: layer.visibleFrom, visibleUntil: layer.visibleUntil, visibleDuration: layer.visibleDuration, time });
+      const offsetProgress = (time - (layer.visibleFrom)) / layer.visibleDuration;
+      // console.log({ offsetProgress });
+      const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1;
+
+      if (shouldDrawLayer) {
+        const rgba = await frameSource.readNextFrame(offsetProgress, canvas);
+        // Frame sources can either render to the provided canvas and return nothing
+        // OR return an raw RGBA blob which will be drawn onto the canvas
+        if (rgba) {
+          // Optimization: Don't need to draw to canvas if there's only one layer
+          if (layerFrameSources.length === 1) return rgba;
 
-        const img = await rgbaToFabricImage({ width, height, rgba });
-        canvas.add(img);
-      } else {
-        // Assume this frame source has drawn its content to the canvas
+          const img = await rgbaToFabricImage({ width, height, rgba });
+          canvas.add(img);
+        } else {
+          // Assume this frame source has drawn its content to the canvas
+        }
       }
     }
     // if (verbose) console.time('Merge frames');
@@ -56,7 +66,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
   }
 
   async function close() {
-    await pMap(layerFrameSources, async (frameSource) => frameSource.close());
+    await pMap(layerFrameSources, async ({ frameSource }) => frameSource.close());
   }
 
   return {

sources/videoFrameSource.js

@@ -1,7 +1,10 @@
 const execa = require('execa');
 const assert = require('assert');
 
-module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, enableFfmpegLog, params }) => {
+const { getFfmpegCommonArgs } = require('../ffmpeg');
+const { readFileStreams } = require('../util');
+
+module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
   const targetSize = width * height * channels;
 
   // TODO assert that we have read the correct amount of frames
@@ -25,11 +28,18 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
   // Cover: https://unix.stackexchange.com/a/192123
   else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`;
 
+  // https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/
+  const streams = await readFileStreams(ffprobePath, path);
+  const firstVideoStream = streams.find((s) => s.codec_type === 'video');
+  // https://superuser.com/a/1116905/658247
+  const inputCodecArgs = ['vp8', 'vp9'].includes(firstVideoStream.codec_name) ? ['-vcodec', 'libvpx'] : [];
+
   // http://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/
   // Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4
   // https://trac.ffmpeg.org/wiki/ChangingFrameRate
   const args = [
-    ...(enableFfmpegLog ? [] : ['-hide_banner', '-loglevel', 'error']),
+    ...getFfmpegCommonArgs({ enableFfmpegLog }),
+    ...inputCodecArgs,
     ...(cutFrom ? ['-ss', cutFrom] : []),
     '-i', path,
     ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []),
@@ -49,17 +59,21 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
   let timeout;
   let ended = false;
 
+  stream.once('end', () => {
+    clearTimeout(timeout);
+    if (verbose) console.log(path, 'ffmpeg video stream ended');
+    ended = true;
+  });
+
   const readNextFrame = () => new Promise((resolve, reject) => {
     if (ended) {
-      console.log(path, 'Tried to read next video frame after ffmpeg stream ended');
+      console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
       resolve();
       return;
     }
     // console.log('Reading new frame', path);
 
     function onEnd() {
-      if (verbose) console.log(path, 'ffmpeg video stream ended');
-      ended = true;
       resolve();
     }
 
@@ -77,7 +91,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
       chunk.copy(buf, length, 0, nCopied);
       length += nCopied;
 
-      if (length > targetSize) console.error('OOPS! Overflow', length);
+      if (length > targetSize) console.error('Video data overflow', length);
 
       if (length >= targetSize) {
         // console.log('Finished reading frame', inFrameCount, path);
@@ -85,7 +99,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
 
         const restLength = chunk.length - nCopied;
         if (restLength > 0) {
-          if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
+          // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
           chunk.slice(nCopied).copy(buf, 0);
           length = restLength;
         } else {
@@ -104,7 +118,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
       console.warn('Timeout on read video frame');
       cleanup();
       resolve();
-    }, 20000);
+    }, 60000);
 
     stream.on('data', handleChunk);
     stream.on('end', onEnd);

transitions.js

@@ -6,7 +6,6 @@ function getRandomTransition() {
   return randomTransitionsSet[Math.floor(Math.random() * randomTransitionsSet.length)];
 }
 
-
 // https://easings.net/
 
 function easeOutExpo(x) {
@@ -17,7 +16,6 @@ function easeInOutCubic(x) {
   return x < 0.5 ? 4 * x * x * x : 1 - ((-2 * x + 2) ** 3) / 2;
 }
 
-
 function getTransitionEasingFunction(easing, transitionName) {
   if (easing !== null) {
     if (easing) return { easeOutExpo }[easing];
@@ -26,8 +24,8 @@ function getTransitionEasingFunction(easing, transitionName) {
   return (progress) => progress;
 }
 
-function calcTransition(defaults, transition) {
-  if (transition === null) return { duration: 0 };
+function calcTransition(defaults, transition, isLastClip) {
+  if (transition === null || isLastClip) return { duration: 0 };
 
   let transitionOrDefault = {
     name: (transition && transition.name) || (defaults.transition && defaults.transition.name),
@@ -63,7 +61,6 @@ function calcTransition(defaults, transition) {
   };
 }
 
-
 module.exports = {
   calcTransition,
   easeInOutCubic,

util.js

@@ -1,4 +1,5 @@
 const execa = require('execa');
+const assert = require('assert');
 
 function parseFps(fps) {
   const match = typeof fps === 'string' && fps.match(/^([0-9]+)\/([0-9]+)$/);
@@ -10,17 +11,31 @@ function parseFps(fps) {
   return undefined;
 }
 
-async function readFileInfo(ffprobePath, p) {
+async function readDuration(ffprobePath, p) {
+  const { stdout } = await execa(ffprobePath, ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', p]);
+  const parsed = parseFloat(stdout);
+  assert(!Number.isNaN(parsed));
+  return parsed;
+}
+
+async function readFileStreams(ffprobePath, p) {
   const { stdout } = await execa(ffprobePath, [
-    '-select_streams', 'v:0', '-show_entries', 'stream', '-of', 'json', p,
+    '-show_entries', 'stream', '-of', 'json', p,
   ]);
   const json = JSON.parse(stdout);
-  const stream = json.streams[0];
+  return json.streams;
+}
+
+async function readVideoFileInfo(ffprobePath, p) {
+  const streams = await readFileStreams(ffprobePath, p);
+  const stream = streams.find((s) => s.codec_type === 'video'); // TODO
+
+  const duration = await readDuration(ffprobePath, p);
 
   const rotation = stream.tags && stream.tags.rotate && parseInt(stream.tags.rotate, 10);
   return {
     // numFrames: parseInt(stream.nb_frames, 10),
-    duration: parseFloat(stream.duration, 10),
+    duration,
     width: stream.width, // TODO coded_width?
     height: stream.height,
     framerateStr: stream.r_frame_rate,
@@ -28,10 +43,61 @@ async function readFileInfo(ffprobePath, p) {
   };
 }
 
+async function readAudioFileInfo(ffprobePath, p) {
+  const duration = await readDuration(ffprobePath, p);
+
+  return { duration };
+}
+
+function toArrayInteger(buffer) {
+  if (buffer.length > 0) {
+    const data = new Uint8ClampedArray(buffer.length);
+    for (let i = 0; i < buffer.length; i += 1) {
+      data[i] = buffer[i];
+    }
+    return data;
+  }
+  return [];
+}
+
 const multipleOf2 = (x) => (x + (x % 2));
 
+function getPositionProps({ position, width, height }) {
+  let originY = 'center';
+  let originX = 'center';
+  let top = height / 2;
+  let left = width / 2;
+
+  const margin = 0.05;
+  if (position === 'top') {
+    originY = 'top';
+    top = height * margin;
+  } else if (position === 'bottom') {
+    originY = 'bottom';
+    top = height * (1 - margin);
+  } else if (position === 'center') {
+    originY = 'center';
+    top = height / 2;
+  }
+
+  if (position && position.x != null) {
+    originX = position.originX || 'left';
+    left = width * position.x;
+  }
+  if (position && position.y != null) {
+    originY = position.originY || 'top';
+    top = height * position.y;
+  }
+
+  return { originX, originY, top, left };
+}
+
 module.exports = {
   parseFps,
-  readFileInfo,
+  readVideoFileInfo,
+  readAudioFileInfo,
   multipleOf2,
+  toArrayInteger,
+  readFileStreams,
+  getPositionProps,
 };