diff --git a/README.md b/README.md index 9a43b94..530f7fd 100644 --- a/README.md +++ b/README.md @@ -151,11 +151,13 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit | `defaults.transition` | | An object `{ name, duration }` describing the default transition. Set to **null** to disable transitions | | | | `defaults.transition.duration` | `--transition-duration` | Default transition duration | `0.5` | sec | | `defaults.transition.name` | `--transition-name` | Default transition type. See **Transition types** | `random` | | -| `clips[]` | | List of clip objects that will be concatenated in sequence | | | -| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the first video layer. | `defaults.duration` | | +| `clips[]` | | List of clip objects that will be played in sequence. Each clip can have one or more layers. | | | +| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the **first video layer**. | `defaults.duration` | | | `clips[].transition` | | Specify transition at the **end** of this clip. See `defaults.transition` | `defaults.transition` | | -| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (last layer on top) | | | +| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (final layer on top) | | | | `clips[].layers[].type` | | Layer type, see below | | | +| `clips[].layers[].visibleFrom` | | What time into the clip should this layer start | | sec | +| `clips[].layers[].visibleUntil` | | What time into the clip should this layer stop | | sec | ### Transition types @@ -180,7 +182,7 @@ For video layers, if parent `clip.duration` is specified, the video will be slow #### Layer type 'audio' -Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to `half speed` and `100x`. +Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to values between `0.5x` and `100x`. | Parameter | Description | Default | | |-|-|-|-| diff --git a/audio.js b/audio.js index b8a2341..b80a6ba 100644 --- a/audio.js +++ b/audio.js @@ -18,7 +18,10 @@ module.exports = ({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose }) => { const segments = await pMap(clips, async (clip, i) => { const clipAudioPath = join(tmpDir, `clip${i}-audio.flac`); - const audioLayers = clip.layers.filter((layer) => ['audio', 'video'].includes(layer.type)); + const audioLayers = clip.layers.filter(({ type, visibleFrom, visibleUntil }) => ( + ['audio', 'video'].includes(type) + // TODO We don't support audio for visibleFrom/visibleUntil layers + && !visibleFrom && visibleUntil == null)); async function createSilence(outPath) { if (verbose) console.log('create silence', clip.duration); diff --git a/examples/smartFit.json5 b/examples/smartFit.json5 new file mode 100644 index 0000000..73b0f28 --- /dev/null +++ b/examples/smartFit.json5 @@ -0,0 +1,13 @@ +{ + // enableFfmpegLog: true, + outPath: './smartFit.mp4', + defaults: { + transition: null, + layer: { backgroundColor: 'white' }, + }, + clips: [ + { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2 }] }, + { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'contain' }] }, + { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'stretch' }] }, + ], +} diff --git a/examples/visibleFromUntil.json5 b/examples/visibleFromUntil.json5 new file mode 100644 index 0000000..7f90f55 --- /dev/null +++ b/examples/visibleFromUntil.json5 @@ -0,0 +1,18 @@ +{ + // enableFfmpegLog: true, + outPath: './visibleFromUntil.mp4', + clips: [ + { duration: 2, layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }, + { type: 'video', path: './assets/dancer1.webm', cutFrom: 0, cutTo: 6, visibleFrom: 0.5, visibleUntil: 1 }, + ] }, + { duration: 2, layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 7.5, cutTo: 10.5 }, + { type: 'news-title', text: 'Hei', visibleFrom: 0.5, visibleUntil: 1 }, + ] }, + { layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 14, cutTo: 18 }, + { type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 1, visibleFrom: 1, visibleUntil: 2 }, + ] }, + ], +} diff --git a/index.js b/index.js index 14b7d3f..3b6110f 100644 --- a/index.js +++ b/index.js @@ -167,6 +167,7 @@ module.exports = async (config = {}) => { return { ...layer, cutFrom, cutTo, audioCutTo, inputDuration, width, height, framerateStr }; } + // Audio is handled later if (type === 'audio') return layer; return handleLayer(layer); @@ -179,14 +180,22 @@ module.exports = async (config = {}) => { assert(clipDuration); // We need to map again, because for audio, we need to know the correct clipDuration - layersOut = await pMap(layersOut, async (layer) => { - const { type, path } = layer; + layersOut = await pMap(layersOut, async (layerIn) => { + const { type, path, visibleUntil, visibleFrom = 0 } = layerIn; + + // This feature allows the user to show another layer overlayed (or replacing) parts of the lower layers (visibleFrom - visibleUntil) + const visibleDuration = ((visibleUntil || clipDuration) - visibleFrom); + assert(visibleDuration > 0 && visibleDuration <= clipDuration, `Invalid visibleFrom ${visibleFrom} or visibleUntil ${visibleUntil}`); + // TODO Also need to handle video layers (framePtsFactor etc) + // TODO handle audio in case of visibleFrom/visibleTo + + const layer = { ...layerIn, visibleFrom, visibleDuration }; if (type === 'audio') { const { duration: fileDuration } = await readAudioFileInfo(ffprobePath, path); let { cutFrom, cutTo } = layer; - console.log({ cutFrom, cutTo, fileDuration, clipDuration }); + // console.log({ cutFrom, cutTo, fileDuration, clipDuration }); if (!cutFrom) cutFrom = 0; cutFrom = Math.max(cutFrom, 0); @@ -410,12 +419,16 @@ module.exports = async (config = {}) => { // eslint-disable-next-line no-constant-condition while (true) { - const fromClipNumFrames = Math.round(getTransitionFromClip().duration * fps); - const toClipNumFrames = getTransitionToClip() && Math.round(getTransitionToClip().duration * fps); + const transitionToClip = getTransitionToClip(); + const transitionFromClip = getTransitionFromClip(); + const fromClipNumFrames = Math.round(transitionFromClip.duration * fps); + const toClipNumFrames = transitionToClip && Math.round(transitionToClip.duration * fps); const fromClipProgress = fromClipFrameAt / fromClipNumFrames; - const toClipProgress = getTransitionToClip() && toClipFrameAt / toClipNumFrames; + const toClipProgress = transitionToClip && toClipFrameAt / toClipNumFrames; + const fromClipTime = transitionFromClip.duration * fromClipProgress; + const toClipTime = transitionToClip && transitionToClip.duration * toClipProgress; - const currentTransition = getTransitionFromClip().transition; + const currentTransition = transitionFromClip.transition; const transitionNumFrames = Math.round(currentTransition.duration * fps); @@ -432,6 +445,7 @@ module.exports = async (config = {}) => { // console.log({ transitionFrameAt, transitionNumFramesSafe }) // const transitionLastFrameIndex = transitionNumFramesSafe - 1; const transitionLastFrameIndex = transitionNumFramesSafe; + // Done with transition? if (transitionFrameAt >= transitionLastFrameIndex) { transitionFromClipId += 1; @@ -454,17 +468,17 @@ module.exports = async (config = {}) => { continue; } - const newFrameSource1Data = await frameSource1.readNextFrame(fromClipProgress); + const newFrameSource1Data = await frameSource1.readNextFrame({ time: fromClipTime }); // If we got no data, use the old data // TODO maybe abort? if (newFrameSource1Data) frameSource1Data = newFrameSource1Data; - else console.log('No frame data returned, using last frame'); + else console.warn('No frame data returned, using last frame'); const isInTransition = frameSource2 && transitionNumFramesSafe > 0 && transitionFrameAt >= 0; let outFrameData; if (isInTransition) { - const frameSource2Data = await frameSource2.readNextFrame(toClipProgress); + const frameSource2Data = await frameSource2.readNextFrame({ time: toClipTime }); if (frameSource2Data) { const progress = transitionFrameAt / transitionNumFramesSafe; @@ -504,9 +518,7 @@ module.exports = async (config = {}) => { console.log(outPath); } catch (err) { console.error('Loop failed', err); - if (outProcess) { - outProcess.kill(); - } + if (outProcess) outProcess.kill(); } finally { if (frameSource1) await frameSource1.close(); if (frameSource2) await frameSource2.close(); diff --git a/sources/frameSource.js b/sources/frameSource.js index 16cdcab..659e6ac 100644 --- a/sources/frameSource.js +++ b/sources/frameSource.js @@ -30,25 +30,33 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver const createFrameSourceFunc = frameSourceFuncs[type]; assert(createFrameSourceFunc, `Invalid type ${type}`); - return createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); + const frameSource = await createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); + return { layer, frameSource }; }, { concurrency: 1 }); - async function readNextFrame(progress) { + async function readNextFrame({ time }) { const canvas = createFabricCanvas({ width, height }); // eslint-disable-next-line no-restricted-syntax - for (const layerFrameSource of layerFrameSources) { - const rgba = await layerFrameSource.readNextFrame(progress, canvas); - // Frame sources can either render to the provided canvas and return nothing - // OR return an raw RGBA blob which will be drawn onto the canvas - if (rgba) { - // Optimization: Don't need to draw to canvas if there's only one layer - if (layerFrameSources.length === 1) return rgba; + for (const { frameSource, layer } of layerFrameSources) { + // console.log({ visibleFrom: layer.visibleFrom, visibleUntil: layer.visibleUntil, visibleDuration: layer.visibleDuration, time }); + const offsetProgress = (time - (layer.visibleFrom)) / layer.visibleDuration; + // console.log({ offsetProgress }); + const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1; - const img = await rgbaToFabricImage({ width, height, rgba }); - canvas.add(img); - } else { - // Assume this frame source has drawn its content to the canvas + if (shouldDrawLayer) { + const rgba = await frameSource.readNextFrame(offsetProgress, canvas); + // Frame sources can either render to the provided canvas and return nothing + // OR return an raw RGBA blob which will be drawn onto the canvas + if (rgba) { + // Optimization: Don't need to draw to canvas if there's only one layer + if (layerFrameSources.length === 1) return rgba; + + const img = await rgbaToFabricImage({ width, height, rgba }); + canvas.add(img); + } else { + // Assume this frame source has drawn its content to the canvas + } } } // if (verbose) console.time('Merge frames'); @@ -57,7 +65,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver } async function close() { - await pMap(layerFrameSources, async (frameSource) => frameSource.close()); + await pMap(layerFrameSources, async ({ frameSource }) => frameSource.close()); } return {