Implement visibleFrom/visibleUntil

README.md

@@ -151,11 +151,13 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit
 | `defaults.transition` | | An object `{ name, duration }` describing the default transition. Set to **null** to disable transitions | | |
 | `defaults.transition.duration` | `--transition-duration` | Default transition duration | `0.5` | sec |
 | `defaults.transition.name` | `--transition-name` | Default transition type. See **Transition types** | `random` | |
-| `clips[]` | | List of clip objects that will be concatenated in sequence | | |
-| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the first video layer. | `defaults.duration` | |
+| `clips[]` | | List of clip objects that will be played in sequence. Each clip can have one or more layers. | | |
+| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the **first video layer**. | `defaults.duration` | |
 | `clips[].transition` | | Specify transition at the **end** of this clip. See `defaults.transition` | `defaults.transition` | |
-| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (last layer on top) | | |
+| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (final layer on top) | | |
 | `clips[].layers[].type` | | Layer type, see below | | |
+| `clips[].layers[].visibleFrom` | | What time into the clip should this layer start | | sec |
+| `clips[].layers[].visibleUntil` | | What time into the clip should this layer stop | | sec |
 
 ### Transition types
 
@@ -180,7 +182,7 @@ For video layers, if parent `clip.duration` is specified, the video will be slow
 
 #### Layer type 'audio'
 
-Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to `half speed` and `100x`.
+Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to values between `0.5x` and `100x`.
 
 | Parameter  | Description | Default | |
 |-|-|-|-|

audio.js

@@ -18,7 +18,10 @@ module.exports = ({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose }) => {
     const segments = await pMap(clips, async (clip, i) => {
       const clipAudioPath = join(tmpDir, `clip${i}-audio.flac`);
 
-      const audioLayers = clip.layers.filter((layer) => ['audio', 'video'].includes(layer.type));
+      const audioLayers = clip.layers.filter(({ type, visibleFrom, visibleUntil }) => (
+        ['audio', 'video'].includes(type)
+        // TODO We don't support audio for visibleFrom/visibleUntil layers
+        && !visibleFrom && visibleUntil == null));
 
       async function createSilence(outPath) {
         if (verbose) console.log('create silence', clip.duration);

examples/smartFit.json5

@@ -0,0 +1,13 @@
+{
+  // enableFfmpegLog: true,
+  outPath: './smartFit.mp4',
+  defaults: {
+    transition: null,
+    layer: { backgroundColor: 'white' },
+  },
+  clips: [
+    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2 }] },
+    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'contain' }] },
+    { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'stretch' }] },
+  ],
+}

examples/visibleFromUntil.json5

@@ -0,0 +1,18 @@
+{
+  // enableFfmpegLog: true,
+  outPath: './visibleFromUntil.mp4',
+  clips: [
+    { duration: 2, layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 },
+      { type: 'video', path: './assets/dancer1.webm', cutFrom: 0, cutTo: 6, visibleFrom: 0.5, visibleUntil: 1 },
+    ] },
+    { duration: 2, layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 7.5, cutTo: 10.5 },
+      { type: 'news-title', text: 'Hei', visibleFrom: 0.5, visibleUntil: 1  },
+    ] },
+    { layers: [
+      { type: 'video', path: './assets/lofoten.mp4', cutFrom: 14, cutTo: 18 },
+      { type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 1, visibleFrom: 1, visibleUntil: 2 },
+    ] },
+  ],
+}

index.js

@@ -167,6 +167,7 @@ module.exports = async (config = {}) => {
         return { ...layer, cutFrom, cutTo, audioCutTo, inputDuration, width, height, framerateStr };
       }
 
+      // Audio is handled later
       if (type === 'audio') return layer;
 
       return handleLayer(layer);
@@ -179,14 +180,22 @@ module.exports = async (config = {}) => {
     assert(clipDuration);
 
     // We need to map again, because for audio, we need to know the correct clipDuration
-    layersOut = await pMap(layersOut, async (layer) => {
-      const { type, path } = layer;
+    layersOut = await pMap(layersOut, async (layerIn) => {
+      const { type, path, visibleUntil, visibleFrom = 0 } = layerIn;
+
+      // This feature allows the user to show another layer overlayed (or replacing) parts of the lower layers (visibleFrom - visibleUntil)
+      const visibleDuration = ((visibleUntil || clipDuration) - visibleFrom);
+      assert(visibleDuration > 0 && visibleDuration <= clipDuration, `Invalid visibleFrom ${visibleFrom} or visibleUntil ${visibleUntil}`);
+      // TODO Also need to handle video layers (framePtsFactor etc)
+      // TODO handle audio in case of visibleFrom/visibleTo
+
+      const layer = { ...layerIn, visibleFrom, visibleDuration };
 
       if (type === 'audio') {
         const { duration: fileDuration } = await readAudioFileInfo(ffprobePath, path);
         let { cutFrom, cutTo } = layer;
 
-        console.log({ cutFrom, cutTo, fileDuration, clipDuration });
+        // console.log({ cutFrom, cutTo, fileDuration, clipDuration });
 
         if (!cutFrom) cutFrom = 0;
         cutFrom = Math.max(cutFrom, 0);
@@ -410,12 +419,16 @@ module.exports = async (config = {}) => {
 
     // eslint-disable-next-line no-constant-condition
     while (true) {
-      const fromClipNumFrames = Math.round(getTransitionFromClip().duration * fps);
-      const toClipNumFrames = getTransitionToClip() && Math.round(getTransitionToClip().duration * fps);
+      const transitionToClip = getTransitionToClip();
+      const transitionFromClip = getTransitionFromClip();
+      const fromClipNumFrames = Math.round(transitionFromClip.duration * fps);
+      const toClipNumFrames = transitionToClip && Math.round(transitionToClip.duration * fps);
       const fromClipProgress = fromClipFrameAt / fromClipNumFrames;
-      const toClipProgress = getTransitionToClip() && toClipFrameAt / toClipNumFrames;
+      const toClipProgress = transitionToClip && toClipFrameAt / toClipNumFrames;
+      const fromClipTime = transitionFromClip.duration * fromClipProgress;
+      const toClipTime = transitionToClip && transitionToClip.duration * toClipProgress;
 
-      const currentTransition = getTransitionFromClip().transition;
+      const currentTransition = transitionFromClip.transition;
 
       const transitionNumFrames = Math.round(currentTransition.duration * fps);
 
@@ -432,6 +445,7 @@ module.exports = async (config = {}) => {
       // console.log({ transitionFrameAt, transitionNumFramesSafe })
       // const transitionLastFrameIndex = transitionNumFramesSafe - 1;
       const transitionLastFrameIndex = transitionNumFramesSafe;
+
       // Done with transition?
       if (transitionFrameAt >= transitionLastFrameIndex) {
         transitionFromClipId += 1;
@@ -454,17 +468,17 @@ module.exports = async (config = {}) => {
         continue;
       }
 
-      const newFrameSource1Data = await frameSource1.readNextFrame(fromClipProgress);
+      const newFrameSource1Data = await frameSource1.readNextFrame({ time: fromClipTime });
       // If we got no data, use the old data
       // TODO maybe abort?
       if (newFrameSource1Data) frameSource1Data = newFrameSource1Data;
-      else console.log('No frame data returned, using last frame');
+      else console.warn('No frame data returned, using last frame');
 
       const isInTransition = frameSource2 && transitionNumFramesSafe > 0 && transitionFrameAt >= 0;
 
       let outFrameData;
       if (isInTransition) {
-        const frameSource2Data = await frameSource2.readNextFrame(toClipProgress);
+        const frameSource2Data = await frameSource2.readNextFrame({ time: toClipTime });
 
         if (frameSource2Data) {
           const progress = transitionFrameAt / transitionNumFramesSafe;
@@ -504,9 +518,7 @@ module.exports = async (config = {}) => {
     console.log(outPath);
   } catch (err) {
     console.error('Loop failed', err);
-    if (outProcess) {
-      outProcess.kill();
-    }
+    if (outProcess) outProcess.kill();
   } finally {
     if (frameSource1) await frameSource1.close();
     if (frameSource2) await frameSource2.close();

sources/frameSource.js

@@ -30,25 +30,33 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
     const createFrameSourceFunc = frameSourceFuncs[type];
     assert(createFrameSourceFunc, `Invalid type ${type}`);
 
-    return createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
+    const frameSource = await createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
+    return { layer, frameSource };
   }, { concurrency: 1 });
 
-  async function readNextFrame(progress) {
+  async function readNextFrame({ time }) {
     const canvas = createFabricCanvas({ width, height });
 
     // eslint-disable-next-line no-restricted-syntax
-    for (const layerFrameSource of layerFrameSources) {
-      const rgba = await layerFrameSource.readNextFrame(progress, canvas);
-      // Frame sources can either render to the provided canvas and return nothing
-      // OR return an raw RGBA blob which will be drawn onto the canvas
-      if (rgba) {
-        // Optimization: Don't need to draw to canvas if there's only one layer
-        if (layerFrameSources.length === 1) return rgba;
+    for (const { frameSource, layer } of layerFrameSources) {
+      // console.log({ visibleFrom: layer.visibleFrom, visibleUntil: layer.visibleUntil, visibleDuration: layer.visibleDuration, time });
+      const offsetProgress = (time - (layer.visibleFrom)) / layer.visibleDuration;
+      // console.log({ offsetProgress });
+      const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1;
 
-        const img = await rgbaToFabricImage({ width, height, rgba });
-        canvas.add(img);
-      } else {
-        // Assume this frame source has drawn its content to the canvas
+      if (shouldDrawLayer) {
+        const rgba = await frameSource.readNextFrame(offsetProgress, canvas);
+        // Frame sources can either render to the provided canvas and return nothing
+        // OR return an raw RGBA blob which will be drawn onto the canvas
+        if (rgba) {
+          // Optimization: Don't need to draw to canvas if there's only one layer
+          if (layerFrameSources.length === 1) return rgba;
+
+          const img = await rgbaToFabricImage({ width, height, rgba });
+          canvas.add(img);
+        } else {
+          // Assume this frame source has drawn its content to the canvas
+        }
       }
     }
     // if (verbose) console.time('Merge frames');
@@ -57,7 +65,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
   }
 
   async function close() {
-    await pMap(layerFrameSources, async (frameSource) => frameSource.close());
+    await pMap(layerFrameSources, async ({ frameSource }) => frameSource.close());
   }
 
   return {