From f1546a02e3a2c1f9c18ff58d944ee4c7d32e3d0b Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 17 Aug 2020 16:54:51 +0200 Subject: [PATCH 01/16] refactoring --- ffmpeg.js | 19 +++++++++++++++++++ index.js | 15 +++++++-------- sources/videoFrameSource.js | 4 +++- 3 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 ffmpeg.js diff --git a/ffmpeg.js b/ffmpeg.js new file mode 100644 index 0000000..eb47ed1 --- /dev/null +++ b/ffmpeg.js @@ -0,0 +1,19 @@ +const fs = require('fs-extra'); + +const getFfmpegCommonArgs = ({ enableFfmpegLog }) => (enableFfmpegLog ? [] : ['-hide_banner', '-loglevel', 'error']); + +const getCutFromArgs = ({ cutFrom }) => (cutFrom ? ['-ss', cutFrom] : []); + +const getCutToArgs = ({ cutTo, cutFrom, framePtsFactor }) => (cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []); + +async function createConcatFile(segments, concatFilePath) { + // https://superuser.com/questions/787064/filename-quoting-in-ffmpeg-concat + await fs.writeFile(concatFilePath, segments.map((seg) => `file '${seg.replace(/'/g, "'\\''")}'`).join('\n')); +} + +module.exports = { + getFfmpegCommonArgs, + getCutFromArgs, + getCutToArgs, + createConcatFile, +}; diff --git a/index.js b/index.js index 075dfe4..6996afc 100644 --- a/index.js +++ b/index.js @@ -21,7 +21,6 @@ const checkTransition = (transition) => assert(transition == null || typeof tran const assertFileExists = async (path) => assert(await fs.exists(path), `File does not exist ${path}`); - module.exports = async (config = {}) => { const { // Testing options: @@ -353,7 +352,7 @@ module.exports = async (config = {}) => { const toClipNumFrames = getTransitionToClip() && Math.round(getTransitionToClip().duration * fps); const fromClipProgress = fromClipFrameCount / fromClipNumFrames; const toClipProgress = getTransitionToClip() && toClipFrameCount / toClipNumFrames; - const frameData1 = await frameSource1.readNextFrame(fromClipProgress); + const frameSource1Data = await frameSource1.readNextFrame(fromClipProgress); const clipTransition = getTransitionFromClip().transition; @@ -371,7 +370,7 @@ module.exports = async (config = {}) => { if (totalFrameCount % 10 === 0) process.stdout.write(`${String(percentDone).padStart(3, ' ')}% `); } - if (!frameData1 || transitionFrameAt >= transitionNumFramesSafe - 1) { + if (!frameSource1Data || transitionFrameAt >= transitionNumFramesSafe - 1) { // if (!frameData1 || transitionFrameAt >= transitionNumFramesSafe) { console.log('Done with transition, switching to next clip'); transitionFromClipId += 1; @@ -393,25 +392,25 @@ module.exports = async (config = {}) => { if (frameSource2 && transitionFrameAt >= 0) { if (verbose) console.log('Transition', 'frame', transitionFrameAt, '/', transitionNumFramesSafe, clipTransition.name, `${clipTransition.duration}s`); - const frameData2 = await frameSource2.readNextFrame(toClipProgress); + const frameSource2Data = await frameSource2.readNextFrame(toClipProgress); toClipFrameCount += 1; - if (frameData2) { + if (frameSource2Data) { const progress = transitionFrameAt / transitionNumFramesSafe; const easedProgress = clipTransition.easingFunction(progress); if (verbose) console.time('runTransitionOnFrame'); - outFrameData = runTransitionOnFrame({ fromFrame: frameData1, toFrame: frameData2, progress: easedProgress, transitionName: clipTransition.name, transitionParams: clipTransition.params }); + outFrameData = runTransitionOnFrame({ fromFrame: frameSource1Data, toFrame: frameSource2Data, progress: easedProgress, transitionName: clipTransition.name, transitionParams: clipTransition.params }); if (verbose) console.timeEnd('runTransitionOnFrame'); } else { console.warn('Got no frame data from clip 2!'); // We have reached end of clip2 but transition is not complete // Pass thru // TODO improve, maybe cut it short - outFrameData = frameData1; + outFrameData = frameSource1Data; } } else { - outFrameData = frameData1; + outFrameData = frameSource1Data; } // If we don't await we get EINVAL when dealing with high resolution files (big writes) diff --git a/sources/videoFrameSource.js b/sources/videoFrameSource.js index f697ac0..0d4b938 100644 --- a/sources/videoFrameSource.js +++ b/sources/videoFrameSource.js @@ -1,6 +1,8 @@ const execa = require('execa'); const assert = require('assert'); +const { getFfmpegCommonArgs } = require('../ffmpeg'); + module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, enableFfmpegLog, params }) => { const targetSize = width * height * channels; @@ -29,7 +31,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg // Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4 // https://trac.ffmpeg.org/wiki/ChangingFrameRate const args = [ - ...(enableFfmpegLog ? [] : ['-hide_banner', '-loglevel', 'error']), + ...getFfmpegCommonArgs({ enableFfmpegLog }), ...(cutFrom ? ['-ss', cutFrom] : []), '-i', path, ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []), From 512ddd6bd16753c80fed65525a5be51606f351b0 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 17 Aug 2020 23:27:23 +0200 Subject: [PATCH 02/16] Implement audio --- README.md | 22 +++++-- audio.js | 135 +++++++++++++++++++++++++++++++++++++++++ examples/alpha.json5 | 14 +++++ examples/audio1.json5 | 24 ++++++++ examples/audio2.json5 | 21 +++++++ index.js | 131 ++++++++++++++++++++++++++++----------- sources/frameSource.js | 13 ++-- transitions.js | 4 +- util.js | 45 ++++++++++++-- 9 files changed, 355 insertions(+), 54 deletions(-) create mode 100644 audio.js create mode 100644 examples/alpha.json5 create mode 100644 examples/audio1.json5 create mode 100644 examples/audio2.json5 diff --git a/README.md b/README.md index 1c3fabd..9a43b94 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This GIF / YouTube was created with this command: "editly [commonFeatures.json5](https://github.com/mifi/editly/blob/master/examples/commonFeatures.json5)". See [more examples here](https://github.com/mifi/editly/tree/master/examples#examples). -**Editly** is a tool and framework for declarative NLE (**non-linear video editing**) using Node.js and ffmpeg. Editly allows you to easily and **programmatically create a video** from a **set of clips, images and titles**, with smooth transitions and music overlaid. +**Editly** is a tool and framework for declarative NLE (**non-linear video editing**) using Node.js and ffmpeg. Editly allows you to easily and **programmatically create a video** from a **set of clips, images, audio and titles**, with smooth transitions and music overlaid. Editly has a simple CLI for quickly assembling a video from a set of clips or images, or you can use its more flexible JavaScript API. @@ -143,7 +143,7 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit | `width` | `--width` | Width which all media will be converted to | `640` | | | `height` | `--height` | Height which all media will be converted to | auto based on `width` and aspect ratio of **first video** | | | `fps` | `--fps` | FPS which all videos will be converted to | First video FPS or `25` | | -| `audioFilePath` | `--audio-file-path` | Set an audio track to the whole output video | | | +| `audioFilePath` | `--audio-file-path` | Set an audio track for the whole video | | | | `fast` | `--fast`, `-f` | Fast mode (low resolution and FPS, useful for getting a quick preview) | `false` | | | `defaults.layer.fontPath` | `--font-path` | Set default font to a .ttf | System font | | | `defaults.layer.*` | | Set any layer parameter that all layers will inherit | | | @@ -152,7 +152,7 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit | `defaults.transition.duration` | `--transition-duration` | Default transition duration | `0.5` | sec | | `defaults.transition.name` | `--transition-name` | Default transition type. See **Transition types** | `random` | | | `clips[]` | | List of clip objects that will be concatenated in sequence | | | -| `clips[].duration` | | Clip duration. See `defaults.duration` | `defaults.duration` | | +| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the first video layer. | `defaults.duration` | | | `clips[].transition` | | Specify transition at the **end** of this clip. See `defaults.transition` | `defaults.transition` | | | `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (last layer on top) | | | | `clips[].layers[].type` | | Layer type, see below | | | @@ -167,15 +167,27 @@ See [examples](https://github.com/mifi/editly/tree/master/examples) and [commonF #### Layer type 'video' -For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. +For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. If the layer has audio, it will be kept (and mixed with other audio layers if present.) | Parameter | Description | Default | | |-|-|-|-| | `path` | Path to video file | | | | `resizeMode` | One of `cover`, `contain`, `stretch` | `contain` | | | `cutFrom` | Time value to cut from | `0` | sec | -| `cutTo` | Time value to cut from | *end of video* | sec | +| `cutTo` | Time value to cut to | *end of video* | sec | | `backgroundColor` | Background of letterboxing | `#000000` | | +| `mixVolume` | Relative volume when mixing this video's audio track with others | `1` | | + +#### Layer type 'audio' + +Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to `half speed` and `100x`. + +| Parameter | Description | Default | | +|-|-|-|-| +| `path` | Path to audio file | | | +| `cutFrom` | Time value to cut from | `0` | sec | +| `cutTo` | Time value to cut to | `clip.duration` | sec | +| `mixVolume` | Relative volume when mixing this audio track with others | `1` | | #### Layer type 'image' diff --git a/audio.js b/audio.js new file mode 100644 index 0000000..941e6bb --- /dev/null +++ b/audio.js @@ -0,0 +1,135 @@ +const pMap = require('p-map'); +const { join, basename } = require('path'); +const execa = require('execa'); +const flatMap = require('lodash/flatMap'); +const fs = require('fs-extra'); + +const { getFfmpegCommonArgs, getCutFromArgs, createConcatFile } = require('./ffmpeg'); +const { readFileStreams } = require('./util'); + +module.exports = ({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose }) => { + async function editAudio({ clips, tmpDir }) { + if (clips.length === 0) return undefined; + + console.log('Extracting audio or creating silence from all clips'); + + const mergedAudioPath = join(tmpDir, 'audio-merged.flac'); + + const segments = await pMap(clips, async (clip, i) => { + const clipAudioPath = join(tmpDir, `clip${i}-audio.flac`); + + const audioLayers = clip.layers.filter((layer) => ['audio', 'video'].includes(layer.type)); + + async function createSilence(outPath) { + if (verbose) console.log('create silence', clip.duration); + const args = [ + '-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=44100', + '-sample_fmt', 's32', + '-ar', '48000', + '-t', clip.duration, + '-c:a', 'flac', + '-y', + outPath, + ]; + await execa(ffmpegPath, args); + } + + if (audioLayers.length > 0) { + const processedAudioLayersRaw = await pMap(audioLayers, async (audioLayer, j) => { + const { path, cutFrom, audioCutTo, framePtsFactor } = audioLayer; + + const streams = await readFileStreams(ffprobePath, path); + if (!streams.some((s) => s.codec_type === 'audio')) return undefined; + + const layerAudioPath = join(tmpDir, `clip${i}-layer${j}-audio.flac`); + + try { + let atempoFilter; + if (Math.abs(framePtsFactor - 1) > 0.01) { + if (verbose) console.log('audio framePtsFactor', framePtsFactor); + const atempo = (1 / framePtsFactor); + if (!(atempo >= 0.5 && atempo <= 100)) { // Required range by ffmpeg + console.warn(`Audio speed ${atempo} is outside accepted range, using silence (clip ${i})`); + return undefined; + } + atempoFilter = `atempo=${atempo}`; + } + + const cutToArg = (audioCutTo - cutFrom) * framePtsFactor; + + const args = [ + ...getFfmpegCommonArgs({ enableFfmpegLog }), + ...getCutFromArgs({ cutFrom }), + '-i', path, + '-t', cutToArg, + '-sample_fmt', 's32', + '-ar', '48000', + '-map', 'a:0', '-c:a', 'flac', + ...(atempoFilter ? ['-filter:a', atempoFilter] : []), + '-y', + layerAudioPath, + ]; + + // console.log(args); + await execa(ffmpegPath, args); + } catch (err) { + if (verbose) console.error('Cannot extract audio from video', path, err); + // Fall back to silence + await createSilence(layerAudioPath); + } + + return { layerAudioPath, audioLayer }; + }, { concurrency: 4 }); + + const processedAudioLayers = processedAudioLayersRaw.filter((p) => p); + + if (processedAudioLayers.length > 1) { + // Merge/mix all layer's audio + + const weights = processedAudioLayers.map(({ audioLayer }) => (audioLayer.mixVolume != null ? audioLayer.mixVolume : 1)); + const args = [ + ...getFfmpegCommonArgs({ enableFfmpegLog }), + ...flatMap(processedAudioLayers, ({ layerAudioPath }) => ['-i', layerAudioPath]), + '-filter_complex', `amix=inputs=${processedAudioLayers.length}:duration=longest:weights=${weights.join(' ')}`, + '-c:a', 'flac', + '-y', + clipAudioPath, + ]; + + await execa(ffmpegPath, args); + } else if (processedAudioLayers.length > 0) { + await fs.rename(processedAudioLayers[0].layerAudioPath, clipAudioPath); + } else { + await createSilence(clipAudioPath); + } + } else { + await createSilence(clipAudioPath); + } + + return basename(clipAudioPath); + }, { concurrency: 4 }); + + const concatFilePath = join(tmpDir, 'audio-segments.txt'); + + console.log('Combining audio', segments, concatFilePath); + + await createConcatFile(segments, concatFilePath); + + const args = [ + ...getFfmpegCommonArgs({ enableFfmpegLog }), + '-f', 'concat', '-safe', '0', + '-i', concatFilePath, + '-c', 'flac', + '-y', + mergedAudioPath, + ]; + await execa(ffmpegPath, args); + + // TODO don't return audio if only silence? + return mergedAudioPath; + } + + return { + editAudio, + }; +}; diff --git a/examples/alpha.json5 b/examples/alpha.json5 new file mode 100644 index 0000000..0598a31 --- /dev/null +++ b/examples/alpha.json5 @@ -0,0 +1,14 @@ +{ + // enableFfmpegLog: true, + outPath: './alpha.mp4', + clips: [ + { duration: 2, layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }, + { type: 'video', path: './assets/dancer1.webm', cutFrom: 0, cutTo: 6 }, + ] }, + { layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }, + { type: 'video', path: './assets/dancer1.webm' }, + ] }, + ], +} diff --git a/examples/audio1.json5 b/examples/audio1.json5 new file mode 100644 index 0000000..d38d974 --- /dev/null +++ b/examples/audio1.json5 @@ -0,0 +1,24 @@ +{ + // enableFfmpegLog: true, + outPath: './audio1.mp4', + defaults: { + transition: null, + }, + clips: [ + { duration: 0.5, layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }] }, + + { layers: [ + { type: 'title', text: 'test' }, + { type: 'audio', path: './assets/High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a', cutFrom: 2, cutTo: 5 }] }, + + { layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 12, cutTo: 14, mixVolume: 0 }, + { type: 'audio', path: './assets/High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a', mixVolume: 0.1 }] }, + + { duration: 2, layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }, + { type: 'audio', path: './assets/High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a', cutFrom: 2, cutTo: 3, mixVolume: 0.5 }] }, + + { duration: 1.8, layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 10, cutTo: 11 }] }, + ], +} diff --git a/examples/audio2.json5 b/examples/audio2.json5 new file mode 100644 index 0000000..fdd0078 --- /dev/null +++ b/examples/audio2.json5 @@ -0,0 +1,21 @@ +{ + // enableFfmpegLog: true, + outPath: './audio2.mp4', + clips: [ + { duration: 0.5, layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }] }, + + { layers: [ + { type: 'title', text: 'test' }, + { type: 'audio', path: './assets/High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a' }] }, + + { layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 12, cutTo: 14, mixVolume: 0.7 }, + { type: 'audio', path: './assets/High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a', mixVolume: 0.3 }] }, + + { layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }, + { type: 'audio', path: './assets/High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a' }] }, + + { layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 10, cutTo: 11 }] }, + ], +} diff --git a/index.js b/index.js index 6996afc..225551e 100644 --- a/index.js +++ b/index.js @@ -1,17 +1,18 @@ const execa = require('execa'); const assert = require('assert'); const pMap = require('p-map'); -const { basename, join } = require('path'); +const { basename, join, dirname } = require('path'); const flatMap = require('lodash/flatMap'); const JSON5 = require('json5'); const fs = require('fs-extra'); -const { parseFps, readFileInfo, multipleOf2 } = require('./util'); +const { parseFps, readVideoFileInfo, readAudioFileInfo, multipleOf2 } = require('./util'); const { registerFont } = require('./sources/fabricFrameSource'); const { createFrameSource } = require('./sources/frameSource'); const { calcTransition } = require('./transitions'); const GlTransitions = require('./glTransitions'); +const Audio = require('./audio'); // Cache const loadedFonts = []; @@ -42,7 +43,8 @@ module.exports = async (config = {}) => { const isGif = outPath.toLowerCase().endsWith('.gif'); - const audioFilePath = isGif ? undefined : audioFilePathIn; + let audioFilePath; + if (!isGif) audioFilePath = audioFilePathIn; if (audioFilePath) await assertFileExists(audioFilePath); @@ -126,66 +128,122 @@ module.exports = async (config = {}) => { } const clips = await pMap(clipsIn, async (clip, clipIndex) => { - const { transition: userTransition, duration: userDuration, layers } = clip; + const { transition: userTransition, duration: userClipDuration, layers } = clip; checkTransition(userTransition); const videoLayers = layers.filter((layer) => layer.type === 'video'); - assert(videoLayers.length <= 1, 'Max 1 video per layer'); - const userOrDefaultDuration = userDuration || defaults.duration; - if (videoLayers.length === 0) assert(userOrDefaultDuration, `Duration is required for clip ${clipIndex}`); + const userClipDurationOrDefault = userClipDuration || defaults.duration; + if (videoLayers.length === 0) assert(userClipDurationOrDefault, `Duration parameter is required for videoless clip ${clipIndex}`); - let duration = userOrDefaultDuration; + const transition = calcTransition(defaults, userTransition, clipIndex === clipsIn.length - 1); - const layersOut = flatMap(await pMap(layers, async (layerIn) => { + let layersOut = flatMap(await pMap(layers, async (layerIn) => { const layer = { ...defaults.layer, ...layerIn }; - const { type } = layer; + const { type, path } = layer; if (type === 'video') { - const { cutFrom: cutFromIn, cutTo: cutToIn, path } = layer; - const fileInfo = await readFileInfo(ffprobePath, path); - const { duration: fileDuration, width: widthIn, height: heightIn, framerateStr, rotation } = fileInfo; - let cutFrom; - let cutTo; - let trimmedSourceDuration = fileDuration; - if (cutFromIn != null || cutToIn != null) { - cutFrom = Math.min(Math.max(0, cutFromIn || 0), fileDuration); - cutTo = Math.min(Math.max(cutFrom, cutToIn || fileDuration), fileDuration); - assert(cutFrom < cutTo, 'cutFrom must be lower than cutTo'); - - trimmedSourceDuration = cutTo - cutFrom; - } + const { duration: fileDuration, width: widthIn, height: heightIn, framerateStr, rotation } = await readVideoFileInfo(ffprobePath, path); + let { cutFrom, cutTo } = layer; + if (!cutFrom) cutFrom = 0; + cutFrom = Math.max(cutFrom, 0); + cutFrom = Math.min(cutFrom, fileDuration); - // If user specified duration, means that should be the output duration - let framePtsFactor; - if (userDuration) { - duration = userDuration; - framePtsFactor = userDuration / trimmedSourceDuration; - } else { - duration = trimmedSourceDuration; - framePtsFactor = 1; - } + if (!cutTo) cutTo = fileDuration; + cutTo = Math.max(cutTo, cutFrom); + cutTo = Math.min(cutTo, fileDuration); + assert(cutFrom < cutTo, 'cutFrom must be lower than cutTo'); + + const inputDuration = cutTo - cutFrom; const isRotated = rotation === 90 || rotation === 270; const width = isRotated ? heightIn : widthIn; const height = isRotated ? widthIn : heightIn; - return { ...layer, cutFrom, cutTo, width, height, framerateStr, framePtsFactor }; + // Compensate for transition duration + const audioCutTo = Math.max(cutFrom, cutTo - transition.duration); + + return { ...layer, cutFrom, cutTo, audioCutTo, inputDuration, width, height, framerateStr }; } + if (type === 'audio') return layer; + return handleLayer(layer); }, { concurrency: 1 })); - const transition = calcTransition(defaults, userTransition); + let clipDuration = userClipDurationOrDefault; + + const firstVideoLayer = layersOut.find((layer) => layer.type === 'video'); + if (firstVideoLayer && !userClipDuration) clipDuration = firstVideoLayer.inputDuration; + assert(clipDuration); + + layersOut = await pMap(layersOut, async (layer) => { + const { type, path } = layer; + + if (type === 'audio') { + const { duration: fileDuration } = await readAudioFileInfo(ffprobePath, path); + let { cutFrom, cutTo } = layer; + + console.log({ cutFrom, cutTo, fileDuration, clipDuration }); + + if (!cutFrom) cutFrom = 0; + cutFrom = Math.max(cutFrom, 0); + cutFrom = Math.min(cutFrom, fileDuration); + + if (!cutTo) cutTo = cutFrom + clipDuration; + cutTo = Math.max(cutTo, cutFrom); + cutTo = Math.min(cutTo, fileDuration); + assert(cutFrom < cutTo, 'cutFrom must be lower than cutTo'); + + const inputDuration = cutTo - cutFrom; + + const framePtsFactor = clipDuration / inputDuration; + + // Compensate for transition duration + const audioCutTo = Math.max(cutFrom, cutTo - transition.duration); + + return { ...layer, cutFrom, cutTo, audioCutTo, framePtsFactor }; + } + + if (layer.type === 'video') { + const { inputDuration } = layer; + + let framePtsFactor; + + // If user explicitly specified duration for clip, it means that should be the output duration of the video + if (userClipDuration) { + // Later we will speed up or slow down video using this factor + framePtsFactor = userClipDuration / inputDuration; + } else { + framePtsFactor = 1; + } + + return { ...layer, framePtsFactor }; + } + + return layer; + }); return { transition, - duration, + duration: clipDuration, layers: layersOut, }; }, { concurrency: 1 }); + const { editAudio } = Audio({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose }); + + const outDir = dirname(outPath); + const tmpDir = join(outDir, 'editly-tmp'); + if (verbose) console.log({ tmpDir }); + await fs.remove(tmpDir); + await fs.mkdirp(tmpDir); + + if (!audioFilePath) { + audioFilePath = await editAudio({ clips, tmpDir }); + } + if (verbose) console.log(JSON5.stringify(clips, null, 2)); // Try to detect parameters from first video @@ -340,7 +398,7 @@ module.exports = async (config = {}) => { const getTransitionFromClip = () => clips[transitionFromClipId]; const getTransitionToClip = () => clips[getTransitionToClipId()]; - const getSource = (clip, clipIndex) => createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, enableFfmpegLog, framerateStr }); + const getSource = (clip, clipIndex) => createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, framerateStr }); const getTransitionToSource = async () => (getTransitionToClip() && getSource(getTransitionToClip(), getTransitionToClipId())); frameSource1 = await getSource(getTransitionFromClip(), transitionFromClipId); @@ -436,6 +494,7 @@ module.exports = async (config = {}) => { } finally { if (frameSource1) await frameSource1.close(); if (frameSource2) await frameSource2.close(); + await fs.remove(tmpDir); } try { diff --git a/sources/frameSource.js b/sources/frameSource.js index 1050659..16cdcab 100644 --- a/sources/frameSource.js +++ b/sources/frameSource.js @@ -5,11 +5,12 @@ const { rgbaToFabricImage, customFabricFrameSource, createCustomCanvasFrameSourc const createVideoFrameSource = require('./videoFrameSource'); const { createGlFrameSource } = require('./glFrameSource'); - -async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, enableFfmpegLog, framerateStr }) { +async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, framerateStr }) { const { layers, duration } = clip; - const layerFrameSources = await pMap(layers, async (layer, layerIndex) => { + const visualLayers = layers.filter((layer) => layer.type !== 'audio'); + + const layerFrameSources = await pMap(visualLayers, async (layer, layerIndex) => { const { type, ...params } = layer; console.log('createFrameSource', type, 'clip', clipIndex, 'layer', layerIndex); @@ -29,15 +30,15 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver const createFrameSourceFunc = frameSourceFuncs[type]; assert(createFrameSourceFunc, `Invalid type ${type}`); - return createFrameSourceFunc({ ffmpegPath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); + return createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); }, { concurrency: 1 }); async function readNextFrame(progress) { const canvas = createFabricCanvas({ width, height }); // eslint-disable-next-line no-restricted-syntax - for (const frameSource of layerFrameSources) { - const rgba = await frameSource.readNextFrame(progress, canvas); + for (const layerFrameSource of layerFrameSources) { + const rgba = await layerFrameSource.readNextFrame(progress, canvas); // Frame sources can either render to the provided canvas and return nothing // OR return an raw RGBA blob which will be drawn onto the canvas if (rgba) { diff --git a/transitions.js b/transitions.js index b32ff1c..45363ee 100644 --- a/transitions.js +++ b/transitions.js @@ -26,8 +26,8 @@ function getTransitionEasingFunction(easing, transitionName) { return (progress) => progress; } -function calcTransition(defaults, transition) { - if (transition === null) return { duration: 0 }; +function calcTransition(defaults, transition, isLastClip) { + if (transition === null || isLastClip) return { duration: 0 }; let transitionOrDefault = { name: (transition && transition.name) || (defaults.transition && defaults.transition.name), diff --git a/util.js b/util.js index 2116187..c4db287 100644 --- a/util.js +++ b/util.js @@ -1,4 +1,5 @@ const execa = require('execa'); +const assert = require('assert'); function parseFps(fps) { const match = typeof fps === 'string' && fps.match(/^([0-9]+)\/([0-9]+)$/); @@ -10,17 +11,31 @@ function parseFps(fps) { return undefined; } -async function readFileInfo(ffprobePath, p) { +async function readDuration(ffprobePath, p) { + const { stdout } = await execa(ffprobePath, ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', p]); + const parsed = parseFloat(stdout); + assert(!Number.isNaN(parsed)); + return parsed; +} + +async function readFileStreams(ffprobePath, p) { const { stdout } = await execa(ffprobePath, [ - '-select_streams', 'v:0', '-show_entries', 'stream', '-of', 'json', p, + '-show_entries', 'stream', '-of', 'json', p, ]); const json = JSON.parse(stdout); - const stream = json.streams[0]; + return json.streams; +} + +async function readVideoFileInfo(ffprobePath, p) { + const streams = await readFileStreams(ffprobePath, p); + const stream = streams.find((s) => s.codec_type === 'video'); // TODO + + const duration = await readDuration(ffprobePath, p); const rotation = stream.tags && stream.tags.rotate && parseInt(stream.tags.rotate, 10); return { // numFrames: parseInt(stream.nb_frames, 10), - duration: parseFloat(stream.duration, 10), + duration, width: stream.width, // TODO coded_width? height: stream.height, framerateStr: stream.r_frame_rate, @@ -28,10 +43,30 @@ async function readFileInfo(ffprobePath, p) { }; } +async function readAudioFileInfo(ffprobePath, p) { + const duration = await readDuration(ffprobePath, p); + + return { duration }; +} + +function toArrayInteger(buffer) { + if (buffer.length > 0) { + const data = new Uint8ClampedArray(buffer.length); + for (let i = 0; i < buffer.length; i += 1) { + data[i] = buffer[i]; + } + return data; + } + return []; +} + const multipleOf2 = (x) => (x + (x % 2)); module.exports = { parseFps, - readFileInfo, + readVideoFileInfo, + readAudioFileInfo, multipleOf2, + toArrayInteger, + readFileStreams, }; From 1b04b7264b6fc3ecc85aa23cd765d8a2db00653e Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 17 Aug 2020 23:27:35 +0200 Subject: [PATCH 03/16] add support for alpha channel --- sources/videoFrameSource.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sources/videoFrameSource.js b/sources/videoFrameSource.js index 0d4b938..3886fa2 100644 --- a/sources/videoFrameSource.js +++ b/sources/videoFrameSource.js @@ -2,8 +2,9 @@ const execa = require('execa'); const assert = require('assert'); const { getFfmpegCommonArgs } = require('../ffmpeg'); +const { readFileStreams } = require('../util'); -module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, enableFfmpegLog, params }) => { +module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => { const targetSize = width * height * channels; // TODO assert that we have read the correct amount of frames @@ -27,11 +28,18 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg // Cover: https://unix.stackexchange.com/a/192123 else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`; + // https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/ + const streams = await readFileStreams(ffprobePath, path); + const firstVideoStream = streams.find((s) => s.codec_type === 'video'); + // https://superuser.com/a/1116905/658247 + const inputCodecArgs = ['vp8', 'vp9'].includes(firstVideoStream.codec_name) ? ['-vcodec', 'libvpx'] : []; + // http://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/ // Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4 // https://trac.ffmpeg.org/wiki/ChangingFrameRate const args = [ ...getFfmpegCommonArgs({ enableFfmpegLog }), + ...inputCodecArgs, ...(cutFrom ? ['-ss', cutFrom] : []), '-i', path, ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []), From 9d125d8ad3cad7743e12bfce855174eed53f5b4a Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Wed, 19 Aug 2020 11:33:45 +0200 Subject: [PATCH 04/16] use absolute paths due to different windows behaviour --- audio.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/audio.js b/audio.js index 941e6bb..b8a2341 100644 --- a/audio.js +++ b/audio.js @@ -1,5 +1,5 @@ const pMap = require('p-map'); -const { join, basename } = require('path'); +const { join, basename, resolve } = require('path'); const execa = require('execa'); const flatMap = require('lodash/flatMap'); const fs = require('fs-extra'); @@ -106,12 +106,13 @@ module.exports = ({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose }) => { await createSilence(clipAudioPath); } - return basename(clipAudioPath); + // https://superuser.com/a/853262/658247 + return resolve(clipAudioPath); }, { concurrency: 4 }); const concatFilePath = join(tmpDir, 'audio-segments.txt'); - console.log('Combining audio', segments, concatFilePath); + console.log('Combining audio', segments.map((s) => basename(s)), concatFilePath); await createConcatFile(segments, concatFilePath); From a0f4e4da6d16f465bf20a39051182e1e45ab6f39 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 23 Aug 2020 22:49:35 +0200 Subject: [PATCH 05/16] Improvements / refactoring Improve timeout logic Hopefully fix dropped frame issue #49 --- examples/timeoutTest.json5 | 7 ++ index.js | 136 ++++++++++++++++++++---------------- sources/videoFrameSource.js | 16 +++-- transitions.js | 3 - 4 files changed, 93 insertions(+), 69 deletions(-) create mode 100644 examples/timeoutTest.json5 diff --git a/examples/timeoutTest.json5 b/examples/timeoutTest.json5 new file mode 100644 index 0000000..616bf6e --- /dev/null +++ b/examples/timeoutTest.json5 @@ -0,0 +1,7 @@ +{ + outPath: './timeoutTest.mp4', + clips: [ + { duration: 1.5, transition: { name: 'crosszoom', duration: 0.3 }, layers: [{ type: 'video', path: './assets/DJI_0156.mov', cutTo: 58 }] }, + { duration: 3, transition: { name: 'fade' }, layers: [{ type: 'video', path: './assets/DJI_0156.mov', cutFrom: 0 }] }, + ], +} diff --git a/index.js b/index.js index 225551e..14b7d3f 100644 --- a/index.js +++ b/index.js @@ -178,6 +178,7 @@ module.exports = async (config = {}) => { if (firstVideoLayer && !userClipDuration) clipDuration = firstVideoLayer.inputDuration; assert(clipDuration); + // We need to map again, because for audio, we need to know the correct clipDuration layersOut = await pMap(layersOut, async (layer) => { const { type, path } = layer; @@ -378,109 +379,124 @@ module.exports = async (config = {}) => { let frameSource1; let frameSource2; + let frameSource1Data; + + let totalFramesWritten = 0; + let fromClipFrameAt = 0; + let toClipFrameAt = 0; + + let transitionFromClipId = 0; + + const getTransitionToClipId = () => transitionFromClipId + 1; + const getTransitionFromClip = () => clips[transitionFromClipId]; + const getTransitionToClip = () => clips[getTransitionToClipId()]; + + const getSource = async (clip, clipIndex) => createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, framerateStr }); + const getTransitionFromSource = async () => getSource(getTransitionFromClip(), transitionFromClipId); + const getTransitionToSource = async () => (getTransitionToClip() && getSource(getTransitionToClip(), getTransitionToClipId())); + try { outProcess = startFfmpegWriterProcess(); let outProcessError; - // If we don't catch it here, the whole process will crash and we cannot process the error + // If we don't handle it here, the whole Node process will crash and we cannot process the error outProcess.stdin.on('error', (err) => { console.error('Output ffmpeg caught error', err); outProcessError = err; }); - let totalFrameCount = 0; - let fromClipFrameCount = 0; - let toClipFrameCount = 0; - - let transitionFromClipId = 0; - - const getTransitionToClipId = () => transitionFromClipId + 1; - const getTransitionFromClip = () => clips[transitionFromClipId]; - const getTransitionToClip = () => clips[getTransitionToClipId()]; - - const getSource = (clip, clipIndex) => createFrameSource({ clip, clipIndex, width, height, channels, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, framerateStr }); - - const getTransitionToSource = async () => (getTransitionToClip() && getSource(getTransitionToClip(), getTransitionToClipId())); - frameSource1 = await getSource(getTransitionFromClip(), transitionFromClipId); + frameSource1 = await getTransitionFromSource(); frameSource2 = await getTransitionToSource(); // eslint-disable-next-line no-constant-condition while (true) { const fromClipNumFrames = Math.round(getTransitionFromClip().duration * fps); const toClipNumFrames = getTransitionToClip() && Math.round(getTransitionToClip().duration * fps); - const fromClipProgress = fromClipFrameCount / fromClipNumFrames; - const toClipProgress = getTransitionToClip() && toClipFrameCount / toClipNumFrames; - const frameSource1Data = await frameSource1.readNextFrame(fromClipProgress); + const fromClipProgress = fromClipFrameAt / fromClipNumFrames; + const toClipProgress = getTransitionToClip() && toClipFrameAt / toClipNumFrames; - const clipTransition = getTransitionFromClip().transition; + const currentTransition = getTransitionFromClip().transition; - const transitionNumFrames = Math.round(clipTransition.duration * fps); + const transitionNumFrames = Math.round(currentTransition.duration * fps); // Each clip has two transitions, make sure we leave enough room: const transitionNumFramesSafe = Math.floor(Math.min(Math.min(fromClipNumFrames, toClipNumFrames != null ? toClipNumFrames : Number.MAX_SAFE_INTEGER) / 2, transitionNumFrames)); // How many frames into the transition are we? negative means not yet started - const transitionFrameAt = fromClipFrameCount - (fromClipNumFrames - transitionNumFramesSafe); - - if (verbose) console.log('Frame', totalFrameCount, 'from', fromClipFrameCount, `(clip ${transitionFromClipId})`, 'to', toClipFrameCount, `(clip ${getTransitionToClipId()})`); + const transitionFrameAt = fromClipFrameAt - (fromClipNumFrames - transitionNumFramesSafe); if (!verbose) { - const percentDone = Math.floor(100 * (totalFrameCount / estimatedTotalFrames)); - if (totalFrameCount % 10 === 0) process.stdout.write(`${String(percentDone).padStart(3, ' ')}% `); + const percentDone = Math.floor(100 * (totalFramesWritten / estimatedTotalFrames)); + if (totalFramesWritten % 10 === 0) process.stdout.write(`${String(percentDone).padStart(3, ' ')}% `); } - if (!frameSource1Data || transitionFrameAt >= transitionNumFramesSafe - 1) { - // if (!frameData1 || transitionFrameAt >= transitionNumFramesSafe) { - console.log('Done with transition, switching to next clip'); + // console.log({ transitionFrameAt, transitionNumFramesSafe }) + // const transitionLastFrameIndex = transitionNumFramesSafe - 1; + const transitionLastFrameIndex = transitionNumFramesSafe; + // Done with transition? + if (transitionFrameAt >= transitionLastFrameIndex) { transitionFromClipId += 1; + console.log(`Done with transition, switching to next transitionFromClip (${transitionFromClipId})`); if (!getTransitionFromClip()) { console.log('No more transitionFromClip, done'); break; } - // Cleanup old, swap and load next + // Cleanup completed frameSource1, swap and load next frameSource2 await frameSource1.close(); frameSource1 = frameSource2; frameSource2 = await getTransitionToSource(); - fromClipFrameCount = transitionNumFramesSafe; - toClipFrameCount = 0; - } else { - let outFrameData; - if (frameSource2 && transitionFrameAt >= 0) { - if (verbose) console.log('Transition', 'frame', transitionFrameAt, '/', transitionNumFramesSafe, clipTransition.name, `${clipTransition.duration}s`); - - const frameSource2Data = await frameSource2.readNextFrame(toClipProgress); - toClipFrameCount += 1; - - if (frameSource2Data) { - const progress = transitionFrameAt / transitionNumFramesSafe; - const easedProgress = clipTransition.easingFunction(progress); - - if (verbose) console.time('runTransitionOnFrame'); - outFrameData = runTransitionOnFrame({ fromFrame: frameSource1Data, toFrame: frameSource2Data, progress: easedProgress, transitionName: clipTransition.name, transitionParams: clipTransition.params }); - if (verbose) console.timeEnd('runTransitionOnFrame'); - } else { - console.warn('Got no frame data from clip 2!'); - // We have reached end of clip2 but transition is not complete - // Pass thru - // TODO improve, maybe cut it short - outFrameData = frameSource1Data; - } + fromClipFrameAt = transitionLastFrameIndex; + toClipFrameAt = 0; + + // eslint-disable-next-line no-continue + continue; + } + + const newFrameSource1Data = await frameSource1.readNextFrame(fromClipProgress); + // If we got no data, use the old data + // TODO maybe abort? + if (newFrameSource1Data) frameSource1Data = newFrameSource1Data; + else console.log('No frame data returned, using last frame'); + + const isInTransition = frameSource2 && transitionNumFramesSafe > 0 && transitionFrameAt >= 0; + + let outFrameData; + if (isInTransition) { + const frameSource2Data = await frameSource2.readNextFrame(toClipProgress); + + if (frameSource2Data) { + const progress = transitionFrameAt / transitionNumFramesSafe; + const easedProgress = currentTransition.easingFunction(progress); + + // if (verbose) console.time('runTransitionOnFrame'); + outFrameData = runTransitionOnFrame({ fromFrame: frameSource1Data, toFrame: frameSource2Data, progress: easedProgress, transitionName: currentTransition.name, transitionParams: currentTransition.params }); + // if (verbose) console.timeEnd('runTransitionOnFrame'); } else { + console.warn('Got no frame data from transitionToClip!'); + // We have probably reached end of clip2 but transition is not complete. Just pass thru clip1 outFrameData = frameSource1Data; } + } else { + // Not in transition. Pass thru clip 1 + outFrameData = frameSource1Data; + } - // If we don't await we get EINVAL when dealing with high resolution files (big writes) - await new Promise((r) => outProcess.stdin.write(outFrameData, () => r())); + if (verbose) { + if (isInTransition) console.log('Writing frame:', totalFramesWritten, 'from clip', transitionFromClipId, `(frame ${fromClipFrameAt})`, 'to clip', getTransitionToClipId(), `(frame ${toClipFrameAt} / ${transitionNumFramesSafe})`, currentTransition.name, `${currentTransition.duration}s`); + else console.log('Writing frame:', totalFramesWritten, 'from clip', transitionFromClipId, `(frame ${fromClipFrameAt})`); + } - if (outProcessError) throw outProcessError; + // If we don't wait for callback, then we get EINVAL when dealing with high resolution files (big writes) + await new Promise((r) => outProcess.stdin.write(outFrameData, () => r())); - fromClipFrameCount += 1; - } + if (outProcessError) throw outProcessError; - totalFrameCount += 1; - } + totalFramesWritten += 1; + fromClipFrameAt += 1; + if (isInTransition) toClipFrameAt += 1; + } // End while loop outProcess.stdin.end(); diff --git a/sources/videoFrameSource.js b/sources/videoFrameSource.js index 3886fa2..6017b08 100644 --- a/sources/videoFrameSource.js +++ b/sources/videoFrameSource.js @@ -59,17 +59,21 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg let timeout; let ended = false; + stream.once('end', () => { + clearTimeout(timeout); + if (verbose) console.log(path, 'ffmpeg video stream ended'); + ended = true; + }); + const readNextFrame = () => new Promise((resolve, reject) => { if (ended) { - console.log(path, 'Tried to read next video frame after ffmpeg stream ended'); + console.log(path, 'Tried to read next video frame after ffmpeg video stream ended'); resolve(); return; } // console.log('Reading new frame', path); function onEnd() { - if (verbose) console.log(path, 'ffmpeg video stream ended'); - ended = true; resolve(); } @@ -87,7 +91,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg chunk.copy(buf, length, 0, nCopied); length += nCopied; - if (length > targetSize) console.error('OOPS! Overflow', length); + if (length > targetSize) console.error('Video data overflow', length); if (length >= targetSize) { // console.log('Finished reading frame', inFrameCount, path); @@ -95,7 +99,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg const restLength = chunk.length - nCopied; if (restLength > 0) { - if (verbose) console.log('Left over data', nCopied, chunk.length, restLength); + // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength); chunk.slice(nCopied).copy(buf, 0); length = restLength; } else { @@ -114,7 +118,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg console.warn('Timeout on read video frame'); cleanup(); resolve(); - }, 20000); + }, 60000); stream.on('data', handleChunk); stream.on('end', onEnd); diff --git a/transitions.js b/transitions.js index 45363ee..9ef86a2 100644 --- a/transitions.js +++ b/transitions.js @@ -6,7 +6,6 @@ function getRandomTransition() { return randomTransitionsSet[Math.floor(Math.random() * randomTransitionsSet.length)]; } - // https://easings.net/ function easeOutExpo(x) { @@ -17,7 +16,6 @@ function easeInOutCubic(x) { return x < 0.5 ? 4 * x * x * x : 1 - ((-2 * x + 2) ** 3) / 2; } - function getTransitionEasingFunction(easing, transitionName) { if (easing !== null) { if (easing) return { easeOutExpo }[easing]; @@ -63,7 +61,6 @@ function calcTransition(defaults, transition, isLastClip) { }; } - module.exports = { calcTransition, easeInOutCubic, From 141786995c500f846217e64bd9b55220b566ba91 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 23 Aug 2020 23:03:35 +0200 Subject: [PATCH 06/16] Implement visibleFrom/visibleUntil --- README.md | 10 +++++---- audio.js | 5 ++++- examples/smartFit.json5 | 13 +++++++++++ examples/visibleFromUntil.json5 | 18 ++++++++++++++++ index.js | 38 ++++++++++++++++++++++----------- sources/frameSource.js | 36 +++++++++++++++++++------------ 6 files changed, 88 insertions(+), 32 deletions(-) create mode 100644 examples/smartFit.json5 create mode 100644 examples/visibleFromUntil.json5 diff --git a/README.md b/README.md index 9a43b94..530f7fd 100644 --- a/README.md +++ b/README.md @@ -151,11 +151,13 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit | `defaults.transition` | | An object `{ name, duration }` describing the default transition. Set to **null** to disable transitions | | | | `defaults.transition.duration` | `--transition-duration` | Default transition duration | `0.5` | sec | | `defaults.transition.name` | `--transition-name` | Default transition type. See **Transition types** | `random` | | -| `clips[]` | | List of clip objects that will be concatenated in sequence | | | -| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the first video layer. | `defaults.duration` | | +| `clips[]` | | List of clip objects that will be played in sequence. Each clip can have one or more layers. | | | +| `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the **first video layer**. | `defaults.duration` | | | `clips[].transition` | | Specify transition at the **end** of this clip. See `defaults.transition` | `defaults.transition` | | -| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (last layer on top) | | | +| `clips[].layers[]` | | List of layers within the current clip that will be overlaid in their natural order (final layer on top) | | | | `clips[].layers[].type` | | Layer type, see below | | | +| `clips[].layers[].visibleFrom` | | What time into the clip should this layer start | | sec | +| `clips[].layers[].visibleUntil` | | What time into the clip should this layer stop | | sec | ### Transition types @@ -180,7 +182,7 @@ For video layers, if parent `clip.duration` is specified, the video will be slow #### Layer type 'audio' -Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to `half speed` and `100x`. +Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`. The slow down/speed-up operation is limited to values between `0.5x` and `100x`. | Parameter | Description | Default | | |-|-|-|-| diff --git a/audio.js b/audio.js index b8a2341..b80a6ba 100644 --- a/audio.js +++ b/audio.js @@ -18,7 +18,10 @@ module.exports = ({ ffmpegPath, ffprobePath, enableFfmpegLog, verbose }) => { const segments = await pMap(clips, async (clip, i) => { const clipAudioPath = join(tmpDir, `clip${i}-audio.flac`); - const audioLayers = clip.layers.filter((layer) => ['audio', 'video'].includes(layer.type)); + const audioLayers = clip.layers.filter(({ type, visibleFrom, visibleUntil }) => ( + ['audio', 'video'].includes(type) + // TODO We don't support audio for visibleFrom/visibleUntil layers + && !visibleFrom && visibleUntil == null)); async function createSilence(outPath) { if (verbose) console.log('create silence', clip.duration); diff --git a/examples/smartFit.json5 b/examples/smartFit.json5 new file mode 100644 index 0000000..73b0f28 --- /dev/null +++ b/examples/smartFit.json5 @@ -0,0 +1,13 @@ +{ + // enableFfmpegLog: true, + outPath: './smartFit.mp4', + defaults: { + transition: null, + layer: { backgroundColor: 'white' }, + }, + clips: [ + { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2 }] }, + { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'contain' }] }, + { layers: [{ type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'stretch' }] }, + ], +} diff --git a/examples/visibleFromUntil.json5 b/examples/visibleFromUntil.json5 new file mode 100644 index 0000000..7f90f55 --- /dev/null +++ b/examples/visibleFromUntil.json5 @@ -0,0 +1,18 @@ +{ + // enableFfmpegLog: true, + outPath: './visibleFromUntil.mp4', + clips: [ + { duration: 2, layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }, + { type: 'video', path: './assets/dancer1.webm', cutFrom: 0, cutTo: 6, visibleFrom: 0.5, visibleUntil: 1 }, + ] }, + { duration: 2, layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 7.5, cutTo: 10.5 }, + { type: 'news-title', text: 'Hei', visibleFrom: 0.5, visibleUntil: 1 }, + ] }, + { layers: [ + { type: 'video', path: './assets/lofoten.mp4', cutFrom: 14, cutTo: 18 }, + { type: 'video', path: './assets/IMG_4605.MOV', cutFrom: 0, cutTo: 1, visibleFrom: 1, visibleUntil: 2 }, + ] }, + ], +} diff --git a/index.js b/index.js index 14b7d3f..3b6110f 100644 --- a/index.js +++ b/index.js @@ -167,6 +167,7 @@ module.exports = async (config = {}) => { return { ...layer, cutFrom, cutTo, audioCutTo, inputDuration, width, height, framerateStr }; } + // Audio is handled later if (type === 'audio') return layer; return handleLayer(layer); @@ -179,14 +180,22 @@ module.exports = async (config = {}) => { assert(clipDuration); // We need to map again, because for audio, we need to know the correct clipDuration - layersOut = await pMap(layersOut, async (layer) => { - const { type, path } = layer; + layersOut = await pMap(layersOut, async (layerIn) => { + const { type, path, visibleUntil, visibleFrom = 0 } = layerIn; + + // This feature allows the user to show another layer overlayed (or replacing) parts of the lower layers (visibleFrom - visibleUntil) + const visibleDuration = ((visibleUntil || clipDuration) - visibleFrom); + assert(visibleDuration > 0 && visibleDuration <= clipDuration, `Invalid visibleFrom ${visibleFrom} or visibleUntil ${visibleUntil}`); + // TODO Also need to handle video layers (framePtsFactor etc) + // TODO handle audio in case of visibleFrom/visibleTo + + const layer = { ...layerIn, visibleFrom, visibleDuration }; if (type === 'audio') { const { duration: fileDuration } = await readAudioFileInfo(ffprobePath, path); let { cutFrom, cutTo } = layer; - console.log({ cutFrom, cutTo, fileDuration, clipDuration }); + // console.log({ cutFrom, cutTo, fileDuration, clipDuration }); if (!cutFrom) cutFrom = 0; cutFrom = Math.max(cutFrom, 0); @@ -410,12 +419,16 @@ module.exports = async (config = {}) => { // eslint-disable-next-line no-constant-condition while (true) { - const fromClipNumFrames = Math.round(getTransitionFromClip().duration * fps); - const toClipNumFrames = getTransitionToClip() && Math.round(getTransitionToClip().duration * fps); + const transitionToClip = getTransitionToClip(); + const transitionFromClip = getTransitionFromClip(); + const fromClipNumFrames = Math.round(transitionFromClip.duration * fps); + const toClipNumFrames = transitionToClip && Math.round(transitionToClip.duration * fps); const fromClipProgress = fromClipFrameAt / fromClipNumFrames; - const toClipProgress = getTransitionToClip() && toClipFrameAt / toClipNumFrames; + const toClipProgress = transitionToClip && toClipFrameAt / toClipNumFrames; + const fromClipTime = transitionFromClip.duration * fromClipProgress; + const toClipTime = transitionToClip && transitionToClip.duration * toClipProgress; - const currentTransition = getTransitionFromClip().transition; + const currentTransition = transitionFromClip.transition; const transitionNumFrames = Math.round(currentTransition.duration * fps); @@ -432,6 +445,7 @@ module.exports = async (config = {}) => { // console.log({ transitionFrameAt, transitionNumFramesSafe }) // const transitionLastFrameIndex = transitionNumFramesSafe - 1; const transitionLastFrameIndex = transitionNumFramesSafe; + // Done with transition? if (transitionFrameAt >= transitionLastFrameIndex) { transitionFromClipId += 1; @@ -454,17 +468,17 @@ module.exports = async (config = {}) => { continue; } - const newFrameSource1Data = await frameSource1.readNextFrame(fromClipProgress); + const newFrameSource1Data = await frameSource1.readNextFrame({ time: fromClipTime }); // If we got no data, use the old data // TODO maybe abort? if (newFrameSource1Data) frameSource1Data = newFrameSource1Data; - else console.log('No frame data returned, using last frame'); + else console.warn('No frame data returned, using last frame'); const isInTransition = frameSource2 && transitionNumFramesSafe > 0 && transitionFrameAt >= 0; let outFrameData; if (isInTransition) { - const frameSource2Data = await frameSource2.readNextFrame(toClipProgress); + const frameSource2Data = await frameSource2.readNextFrame({ time: toClipTime }); if (frameSource2Data) { const progress = transitionFrameAt / transitionNumFramesSafe; @@ -504,9 +518,7 @@ module.exports = async (config = {}) => { console.log(outPath); } catch (err) { console.error('Loop failed', err); - if (outProcess) { - outProcess.kill(); - } + if (outProcess) outProcess.kill(); } finally { if (frameSource1) await frameSource1.close(); if (frameSource2) await frameSource2.close(); diff --git a/sources/frameSource.js b/sources/frameSource.js index 16cdcab..659e6ac 100644 --- a/sources/frameSource.js +++ b/sources/frameSource.js @@ -30,25 +30,33 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver const createFrameSourceFunc = frameSourceFuncs[type]; assert(createFrameSourceFunc, `Invalid type ${type}`); - return createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); + const frameSource = await createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); + return { layer, frameSource }; }, { concurrency: 1 }); - async function readNextFrame(progress) { + async function readNextFrame({ time }) { const canvas = createFabricCanvas({ width, height }); // eslint-disable-next-line no-restricted-syntax - for (const layerFrameSource of layerFrameSources) { - const rgba = await layerFrameSource.readNextFrame(progress, canvas); - // Frame sources can either render to the provided canvas and return nothing - // OR return an raw RGBA blob which will be drawn onto the canvas - if (rgba) { - // Optimization: Don't need to draw to canvas if there's only one layer - if (layerFrameSources.length === 1) return rgba; + for (const { frameSource, layer } of layerFrameSources) { + // console.log({ visibleFrom: layer.visibleFrom, visibleUntil: layer.visibleUntil, visibleDuration: layer.visibleDuration, time }); + const offsetProgress = (time - (layer.visibleFrom)) / layer.visibleDuration; + // console.log({ offsetProgress }); + const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1; - const img = await rgbaToFabricImage({ width, height, rgba }); - canvas.add(img); - } else { - // Assume this frame source has drawn its content to the canvas + if (shouldDrawLayer) { + const rgba = await frameSource.readNextFrame(offsetProgress, canvas); + // Frame sources can either render to the provided canvas and return nothing + // OR return an raw RGBA blob which will be drawn onto the canvas + if (rgba) { + // Optimization: Don't need to draw to canvas if there's only one layer + if (layerFrameSources.length === 1) return rgba; + + const img = await rgbaToFabricImage({ width, height, rgba }); + canvas.add(img); + } else { + // Assume this frame source has drawn its content to the canvas + } } } // if (verbose) console.time('Merge frames'); @@ -57,7 +65,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver } async function close() { - await pMap(layerFrameSources, async (frameSource) => frameSource.close()); + await pMap(layerFrameSources, async ({ frameSource }) => frameSource.close()); } return { From d8f9546a2c3c595ff5298d024609c5bc6cc7ee5f Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 23 Aug 2020 23:03:43 +0200 Subject: [PATCH 07/16] fix readme bug --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index fc59fd3..856d800 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,7 +14,7 @@ editly kenBurns.json5 ![](https://github.com/mifi/gifs/raw/master/newsTitle.gif) -[kenBurns.json5](https://github.com/mifi/editly/blob/master/examples/newsTitle.json5) +[newsTitle.json5](https://github.com/mifi/editly/blob/master/examples/newsTitle.json5) ```bash editly newsTitle.json5 From 6c3619494da7707788f163a8ac3534371a6c2fc1 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 12:24:38 +0200 Subject: [PATCH 08/16] add flag keepSourceAudio --- README.md | 2 ++ cli.js | 5 ++++- index.js | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 530f7fd..1659455 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit } }, audioFilePath, + keepSourceAudio, clips: [ { transition, @@ -144,6 +145,7 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit | `height` | `--height` | Height which all media will be converted to | auto based on `width` and aspect ratio of **first video** | | | `fps` | `--fps` | FPS which all videos will be converted to | First video FPS or `25` | | | `audioFilePath` | `--audio-file-path` | Set an audio track for the whole video | | | +| `keepSourceAudio` | `--keep-source-audio` | Keep audio from source files | | | | `fast` | `--fast`, `-f` | Fast mode (low resolution and FPS, useful for getting a quick preview) | `false` | | | `defaults.layer.fontPath` | `--font-path` | Set default font to a .ttf | System font | | | `defaults.layer.*` | | Set any layer parameter that all layers will inherit | | | diff --git a/cli.js b/cli.js index 0a9134d..83ebd1b 100644 --- a/cli.js +++ b/cli.js @@ -32,6 +32,7 @@ const cli = meow(` --fps FPS which all videos will be converted to --font-path Set default font to a .ttf --audio-file-path Add an audio track + --keep-source-audio Keep audio from source files --fast, -f Fast mode (low resolution and FPS, useful for getting a quick preview) --verbose, -v @@ -45,6 +46,7 @@ const cli = meow(` `, { flags: { verbose: { type: 'boolean', alias: 'v' }, + keepSourceAudio: { type: 'boolean' }, fast: { type: 'boolean', alias: 'f' }, transitionDuration: { type: 'number' }, clipDuration: { type: 'number' }, @@ -92,7 +94,7 @@ const cli = meow(` params.clips = clips.map((clip) => ({ layers: [clip] })); } - const { verbose, transitionName, transitionDuration, clipDuration, width, height, fps, audioFilePath, fontPath, fast, out: outPath } = cli.flags; + const { verbose, transitionName, transitionDuration, clipDuration, width, height, fps, audioFilePath, fontPath, fast, out: outPath, keepSourceAudio } = cli.flags; if (transitionName || transitionDuration != null) { params.defaults.transition = {}; @@ -110,6 +112,7 @@ const cli = meow(` if (outPath) params.outPath = outPath; if (audioFilePath) params.audioFilePath = audioFilePath; + if (keepSourceAudio) params.keepSourceAudio = true; if (width) params.width = width; if (height) params.height = height; if (fps) params.fps = fps; diff --git a/index.js b/index.js index 3b6110f..22e8663 100644 --- a/index.js +++ b/index.js @@ -36,6 +36,7 @@ module.exports = async (config = {}) => { fps: requestedFps, defaults: defaultsIn = {}, audioFilePath: audioFilePathIn, + keepSourceAudio, ffmpegPath = 'ffmpeg', ffprobePath = 'ffprobe', @@ -250,7 +251,7 @@ module.exports = async (config = {}) => { await fs.remove(tmpDir); await fs.mkdirp(tmpDir); - if (!audioFilePath) { + if (!audioFilePath && keepSourceAudio) { audioFilePath = await editAudio({ clips, tmpDir }); } From b1af8526b01731df833d93c3865c32aeac186035 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 15:57:38 +0200 Subject: [PATCH 09/16] add better positoion option for title --- README.md | 4 +++- examples/subtitle.json5 | 4 ++++ sources/fabricFrameSource.js | 41 ++++++++++++++++++++++++++---------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1659455..70cc9b9 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,9 @@ Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting - `fontPath` - See `defaults.layer.fontPath` - `text` - Title text to show, keep it short - `textColor` - default `#ffffff` -- `position` - Vertical position: `top`, `bottom` or `center` +- `position` - One of either: + - `top`, `bottom` or `center` - vertical position + - An object `{ x, y, originX = 'left', originY = 'top' }`, where `{ x: 0, y: 0 }` is the upper left corner of the screen, and `{ x: 1, y: 1 }` is the lower right corner. `originX` and `originY` are optional, and specify the position origin of the text object. #### Layer type 'subtitle' - `fontPath` - See `defaults.layer.fontPath` diff --git a/examples/subtitle.json5 b/examples/subtitle.json5 index b1d2207..3bd711b 100644 --- a/examples/subtitle.json5 +++ b/examples/subtitle.json5 @@ -9,5 +9,9 @@ { type: 'subtitle', text: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident.' }, { type: 'title', position: 'top', text: 'Subtitles' }, ] }, + { duration: 2, layers: [ + { type: 'fill-color' }, + { type: 'title', position: { x: 0, y: 1, originY: 'bottom' }, text: 'Custom position' }, + ] }, ], } \ No newline at end of file diff --git a/sources/fabricFrameSource.js b/sources/fabricFrameSource.js index 7466ed1..48e2de5 100644 --- a/sources/fabricFrameSource.js +++ b/sources/fabricFrameSource.js @@ -232,6 +232,32 @@ async function subtitleFrameSource({ width, height, params }) { return { onRender }; } +function getPositionProps({ position, width, height, objHeight }) { + let originY = 'center'; + let originX = 'center'; + let top = height / 2; + let left = width / 2; + + if (position === 'top') { + originY = 'top'; + top = height * objHeight; + } else if (position === 'bottom') { + originY = 'bottom'; + top = height; + } + + if (position.x != null) { + originX = position.originX || 'left'; + left = width * position.x; + } + if (position.y != null) { + originY = position.originY || 'top'; + top = height * position.y; + } + + return { originX, originY, top, left }; +} + async function titleFrameSource({ width, height, params }) { const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center' } = params; @@ -252,22 +278,15 @@ async function titleFrameSource({ width, height, params }) { width: width * 0.8, }); + // We need the text as an image in order to scale it const textImage = await new Promise((r) => textBox.cloneAsImage(r)); - let originY = 'center'; - let top = height / 2; - if (position === 'top') { - originY = 'top'; - top = height * 0.05; - } else if (position === 'bottom') { - originY = 'bottom'; - top = height; - } + const { left, top, originX, originY } = getPositionProps({ position, width, height, objHeight: 0.05 }); textImage.set({ - originX: 'center', + originX, originY, - left: width / 2, + left, top, scaleX: scale, scaleY: scale, From 6501497ca30f930fd18d2b106ab549d5f3d61c3c Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 20:40:15 +0200 Subject: [PATCH 10/16] add layerType defaults --- README.md | 6 ++++++ examples/subtitle.json5 | 1 + index.js | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 70cc9b9..8b90f9a 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,13 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit layer: { fontPath, // ...more layer defaults + }, + layerType: { + 'fill-color': { + color: '#ff6666', } + // ...more per-layer-type defaults + }, }, audioFilePath, keepSourceAudio, diff --git a/examples/subtitle.json5 b/examples/subtitle.json5 index 3bd711b..0e31209 100644 --- a/examples/subtitle.json5 +++ b/examples/subtitle.json5 @@ -2,6 +2,7 @@ outPath: './subtitle.mp4', defaults: { layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' }, + layerType: { 'fill-color': { color: '#00aa00' } } }, clips: [ { duration: 2, layers: [ diff --git a/index.js b/index.js index 22e8663..8e8f72a 100644 --- a/index.js +++ b/index.js @@ -141,7 +141,9 @@ module.exports = async (config = {}) => { const transition = calcTransition(defaults, userTransition, clipIndex === clipsIn.length - 1); let layersOut = flatMap(await pMap(layers, async (layerIn) => { - const layer = { ...defaults.layer, ...layerIn }; + const globalLayerDefaults = defaults.layer || {}; + const thisLayerDefaults = (defaults.layerType || {})[layerIn.type]; + const layer = { ...globalLayerDefaults, ...thisLayerDefaults, ...layerIn }; const { type, path } = layer; if (type === 'video') { From 720768efe1e6ab87e5669ae274f35799d0e382e5 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 20:53:29 +0200 Subject: [PATCH 11/16] - Implemet image overlay - add zoomDirection and zoomAmount to title --- README.md | 42 +++++++++++++++--- examples/subtitle.json5 | 2 +- index.js | 4 +- sources/fabricFrameSource.js | 82 +++++++++++++++++++++--------------- sources/frameSource.js | 3 +- util.js | 31 ++++++++++++++ 6 files changed, 119 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 8b90f9a..23473b7 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit layerType: { 'fill-color': { color: '#ff6666', - } + } // ...more per-layer-type defaults }, }, @@ -158,7 +158,7 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit | `defaults.duration` | `--clip-duration` | Set default clip duration for clips that don't have an own duration | `4` | sec | | `defaults.transition` | | An object `{ name, duration }` describing the default transition. Set to **null** to disable transitions | | | | `defaults.transition.duration` | `--transition-duration` | Default transition duration | `0.5` | sec | -| `defaults.transition.name` | `--transition-name` | Default transition type. See **Transition types** | `random` | | +| `defaults.transition.name` | `--transition-name` | Default transition type. See [Transition types](#transition-types) | `random` | | | `clips[]` | | List of clip objects that will be played in sequence. Each clip can have one or more layers. | | | | `clips[].duration` | | Clip duration. See `defaults.duration`. If unset, the clip duration will be that of the **first video layer**. | `defaults.duration` | | | `clips[].transition` | | Specify transition at the **end** of this clip. See `defaults.transition` | `defaults.transition` | | @@ -201,19 +201,32 @@ Audio layers will be mixed together. If `cutFrom`/`cutTo` is set, the resulting #### Layer type 'image' +Full screen image (auto letterboxed) + | Parameter | Description | Default | | |-|-|-|-| | `path` | Path to image file | | | -| `zoomDirection` | Zoom direction for Ken Burns effect: `in`, `out` or `null` to disable | `in` | | -| `zoomAmount` | Zoom amount for Ken Burns effect | `0.1` | | + +See also See [Ken Burns parameters](#ken-burns-parameters). + +#### Layer type 'image-overlay' + +Image overlay with a custom position on the screen. + +| Parameter | Description | Default | | +|-|-|-|-| +| `path` | Path to image file | | | +| `position` | See [Position parameter](#position-parameter) | | | + +See also [Ken Burns parameters](#ken-burns-parameters). #### Layer type 'title' - `fontPath` - See `defaults.layer.fontPath` - `text` - Title text to show, keep it short - `textColor` - default `#ffffff` -- `position` - One of either: - - `top`, `bottom` or `center` - vertical position - - An object `{ x, y, originX = 'left', originY = 'top' }`, where `{ x: 0, y: 0 }` is the upper left corner of the screen, and `{ x: 1, y: 1 }` is the lower right corner. `originX` and `originY` are optional, and specify the position origin of the text object. +- `position` - See **Positions** + +See also [Ken Burns parameters](#ken-burns-parameters) #### Layer type 'subtitle' - `fontPath` - See `defaults.layer.fontPath` @@ -261,6 +274,21 @@ Loads a GLSL shader. See [gl.json5](https://github.com/mifi/editly/blob/master/e - `fragmentPath` - `vertexPath` (optional) +### Position parameter + +Certain layers support the position parameter + +`position` can be one of either: + - `top`, `bottom` or `center` - vertical position (horizontally centered) + - An object `{ x, y, originX = 'left', originY = 'top' }`, where `{ x: 0, y: 0 }` is the upper left corner of the screen, and `{ x: 1, y: 1 }` is the lower right corner, `x` is relative to video width, `y` to height. `originX` and `originY` are optional, and specify the position's origin (anchor position) of the object. + +### Ken Burns parameters + +| Parameter | Description | Default | | +|-|-|-|-| +| `zoomDirection` | Zoom direction for Ken Burns effect: `in`, `out` or `null` to disable | | | +| `zoomAmount` | Zoom amount for Ken Burns effect | `0.1` | | + ## Troubleshooting - If you get `Error: The specified module could not be found.`, try: `npm un -g editly && npm i -g --build-from-source editly` (see [#15](https://github.com/mifi/editly/issues/15)) diff --git a/examples/subtitle.json5 b/examples/subtitle.json5 index 0e31209..5641fe3 100644 --- a/examples/subtitle.json5 +++ b/examples/subtitle.json5 @@ -12,7 +12,7 @@ ] }, { duration: 2, layers: [ { type: 'fill-color' }, - { type: 'title', position: { x: 0, y: 1, originY: 'bottom' }, text: 'Custom position' }, + { type: 'title', position: { x: 0, y: 1, originY: 'bottom' }, text: 'Custom position', zoomDirection: null }, ] }, ], } \ No newline at end of file diff --git a/index.js b/index.js index 8e8f72a..532715f 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,7 @@ module.exports = async (config = {}) => { const { type, ...restLayer } = layer; // https://github.com/mifi/editly/issues/39 - if (type === 'image') { + if (['image', 'image-overlay'].includes(type)) { await assertFileExists(restLayer.path); } else if (type === 'gl') { await assertFileExists(restLayer.fragmentPath); @@ -78,7 +78,7 @@ module.exports = async (config = {}) => { if (['fabric', 'canvas'].includes(type)) assert(typeof layer.func === 'function', '"func" must be a function'); - if (['image', 'fabric', 'canvas', 'gl', 'radial-gradient', 'linear-gradient', 'fill-color'].includes(type)) return layer; + if (['image', 'image-overlay', 'fabric', 'canvas', 'gl', 'radial-gradient', 'linear-gradient', 'fill-color'].includes(type)) return layer; // TODO if random-background radial-gradient linear etc if (type === 'pause') return handleLayer({ ...restLayer, type: 'fill-color' }); diff --git a/sources/fabricFrameSource.js b/sources/fabricFrameSource.js index 48e2de5..8b8211e 100644 --- a/sources/fabricFrameSource.js +++ b/sources/fabricFrameSource.js @@ -7,6 +7,7 @@ const { createCanvas } = nodeCanvas; const { canvasToRgba } = require('./shared'); const { getRandomGradient, getRandomColors } = require('../colors'); const { easeOutExpo } = require('../transitions'); +const { getPositionProps } = require('../util'); // http://fabricjs.com/kitchensink @@ -56,10 +57,21 @@ async function createFabricFrameSource(func, { width, height, ...rest }) { }; } +const loadImage = async (path) => new Promise((resolve) => fabric.util.loadImage(fileUrl(path), resolve)); + +function getZoomParams({ progress, zoomDirection, zoomAmount }) { + let scaleFactor = 1; + if (zoomDirection === 'in') scaleFactor = (1 + zoomAmount * progress); + else if (zoomDirection === 'out') scaleFactor = (1 + zoomAmount * (1 - progress)); + return scaleFactor; +} + async function imageFrameSource({ verbose, params, width, height }) { - if (verbose) console.log('Loading', params.path); + const { path, zoomDirection = 'in', zoomAmount = 0.1 } = params; - const imgData = await new Promise((resolve) => fabric.util.loadImage(fileUrl(params.path), resolve)); + if (verbose) console.log('Loading', path); + + const imgData = await loadImage(path); const getImg = () => new fabric.Image(imgData, { originX: 'center', @@ -76,15 +88,10 @@ async function imageFrameSource({ verbose, params, width, height }) { if (blurredImg.height > blurredImg.width) blurredImg.scaleToWidth(width); else blurredImg.scaleToHeight(height); - async function onRender(progress, canvas) { - const { zoomDirection = 'in', zoomAmount = 0.1 } = params; - const img = getImg(); - let scaleFactor = 1; - if (zoomDirection === 'in') scaleFactor = (1 + progress * zoomAmount); - else if (zoomDirection === 'out') scaleFactor = (1 + zoomAmount * (1 - progress)); + const scaleFactor = getZoomParams({ progress, zoomDirection, zoomAmount }); if (img.height > img.width) img.scaleToHeight(height * scaleFactor); else img.scaleToWidth(width * scaleFactor); @@ -232,34 +239,40 @@ async function subtitleFrameSource({ width, height, params }) { return { onRender }; } -function getPositionProps({ position, width, height, objHeight }) { - let originY = 'center'; - let originX = 'center'; - let top = height / 2; - let left = width / 2; - - if (position === 'top') { - originY = 'top'; - top = height * objHeight; - } else if (position === 'bottom') { - originY = 'bottom'; - top = height; - } +async function imageOverlayFrameSource({ params, width, height }) { + const { path, position, width: relWidth, height: relHeight, zoomDirection, zoomAmount = 0.1 } = params; - if (position.x != null) { - originX = position.originX || 'left'; - left = width * position.x; - } - if (position.y != null) { - originY = position.originY || 'top'; - top = height * position.y; + const imgData = await loadImage(path); + + const { left, top, originX, originY } = getPositionProps({ position, width, height }); + + const img = new fabric.Image(imgData, { + originX, + originY, + left, + top, + }); + + async function onRender(progress, canvas) { + const scaleFactor = getZoomParams({ progress, zoomDirection, zoomAmount }); + + if (relWidth != null) { + img.scaleToWidth(relWidth * width * scaleFactor); + } else if (relHeight != null) { + img.scaleToHeight(relHeight * height * scaleFactor); + } else { + // Default to screen width + img.scaleToWidth(width * scaleFactor); + } + + canvas.add(img); } - return { originX, originY, top, left }; + return { onRender }; } async function titleFrameSource({ width, height, params }) { - const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center' } = params; + const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center', zoomDirection = 'in', zoomAmount = 0.2 } = params; async function onRender(progress, canvas) { // console.log('progress', progress); @@ -268,7 +281,7 @@ async function titleFrameSource({ width, height, params }) { const fontSize = Math.round(min * 0.1); - const scale = (1 + progress * 0.2).toFixed(4); + const scaleFactor = getZoomParams({ progress, zoomDirection, zoomAmount }); const textBox = new fabric.Textbox(text, { fill: textColor, @@ -281,15 +294,15 @@ async function titleFrameSource({ width, height, params }) { // We need the text as an image in order to scale it const textImage = await new Promise((r) => textBox.cloneAsImage(r)); - const { left, top, originX, originY } = getPositionProps({ position, width, height, objHeight: 0.05 }); + const { left, top, originX, originY } = getPositionProps({ position, width, height }); textImage.set({ originX, originY, left, top, - scaleX: scale, - scaleY: scale, + scaleX: scaleFactor, + scaleY: scaleFactor, }); canvas.add(textImage); } @@ -382,6 +395,7 @@ module.exports = { radialGradientFrameSource, linearGradientFrameSource, imageFrameSource, + imageOverlayFrameSource, createFabricCanvas, renderFabricCanvas, diff --git a/sources/frameSource.js b/sources/frameSource.js index 659e6ac..dd1dc96 100644 --- a/sources/frameSource.js +++ b/sources/frameSource.js @@ -1,7 +1,7 @@ const assert = require('assert'); const pMap = require('p-map'); -const { rgbaToFabricImage, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource, newsTitleFrameSource, createFabricCanvas, renderFabricCanvas } = require('./fabricFrameSource'); +const { rgbaToFabricImage, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, imageOverlayFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource, newsTitleFrameSource, createFabricCanvas, renderFabricCanvas } = require('./fabricFrameSource'); const createVideoFrameSource = require('./videoFrameSource'); const { createGlFrameSource } = require('./glFrameSource'); @@ -20,6 +20,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver canvas: createCustomCanvasFrameSource, fabric: async (opts) => createFabricFrameSource(customFabricFrameSource, opts), image: async (opts) => createFabricFrameSource(imageFrameSource, opts), + 'image-overlay': async (opts) => createFabricFrameSource(imageOverlayFrameSource, opts), title: async (opts) => createFabricFrameSource(titleFrameSource, opts), subtitle: async (opts) => createFabricFrameSource(subtitleFrameSource, opts), 'linear-gradient': async (opts) => createFabricFrameSource(linearGradientFrameSource, opts), diff --git a/util.js b/util.js index c4db287..5933143 100644 --- a/util.js +++ b/util.js @@ -62,6 +62,36 @@ function toArrayInteger(buffer) { const multipleOf2 = (x) => (x + (x % 2)); +function getPositionProps({ position, width, height }) { + let originY = 'center'; + let originX = 'center'; + let top = height / 2; + let left = width / 2; + + const margin = 0.05; + if (position === 'top') { + originY = 'top'; + top = height * margin; + } else if (position === 'bottom') { + originY = 'bottom'; + top = height * (1 - margin); + } else if (position === 'center') { + originY = 'center'; + top = height / 2; + } + + if (position && position.x != null) { + originX = position.originX || 'left'; + left = width * position.x; + } + if (position && position.y != null) { + originY = position.originY || 'top'; + top = height * position.y; + } + + return { originX, originY, top, left }; +} + module.exports = { parseFps, readVideoFileInfo, @@ -69,4 +99,5 @@ module.exports = { multipleOf2, toArrayInteger, readFileStreams, + getPositionProps, }; From d28436250b43fbfc14f208025b9e0ab7c0eb69f2 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 21:06:54 +0200 Subject: [PATCH 12/16] update readme --- README.md | 6 +++++- examples/README.md | 41 +++++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 23473b7..0aafd08 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,16 @@ Inspired by [ffmpeg-concat](https://github.com/transitive-bullshit/ffmpeg-concat - Accepts custom HTML5 Canvas / Fabric.js JavaScript code for custom screens or dynamic overlays - Render custom GL shaders (for example from [shadertoy](https://www.shadertoy.com/)) - Can output GIF +- Preserve audio sources or mix multiple +- Overlay transparent images or videos +- Show different sub-clips for parts of a clips duration (B-roll) ## Use cases - Create a slideshow from a set of pictures with text overlay - Create a fast-paced trailer or promo video - Create a tutorial video with help text +- Create news stories - Simply convert a video to a GIF - Resize video to any size or framerate and with automatic letterboxing/cropping (e.g. if you need to upload a video somewhere but the site complains `Video must be 1337x1000 30fps`) @@ -119,7 +123,7 @@ Edit specs are JavaScript / JSON objects describing the whole edit operation wit }, }, audioFilePath, - keepSourceAudio, + keepSourceAudio: false, clips: [ { transition, diff --git a/examples/README.md b/examples/README.md index 856d800..32fef3b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,54 +1,52 @@ # Examples -## Ken Burns zoom slideshow +## Image slideshow with Ken Burns zoom ![](https://github.com/mifi/gifs/raw/master/kenburns.gif) [kenBurns.json5](https://github.com/mifi/editly/blob/master/examples/kenBurns.json5) -```bash -editly kenBurns.json5 -``` - ## News title ![](https://github.com/mifi/gifs/raw/master/newsTitle.gif) [newsTitle.json5](https://github.com/mifi/editly/blob/master/examples/newsTitle.json5) -```bash -editly newsTitle.json5 -``` - ## Resize modes ![](https://github.com/mifi/gifs/raw/master/resizeHorizontal.gif) [resizeHorizontal.json5](https://github.com/mifi/editly/blob/master/examples/resizeHorizontal.json5) -```bash -editly resizeHorizontal.json5 -``` - ## Speed up / slow down with cutting ![](https://github.com/mifi/gifs/raw/master/speedTest.gif) [speedTest.json5](https://github.com/mifi/editly/blob/master/examples/speedTest.json5) -```bash -editly speedTest.json5 -``` - ## Title and subtitle ![](https://github.com/mifi/gifs/raw/master/subtitle.gif) [subtitle.json5](https://github.com/mifi/editly/blob/master/examples/subtitle.json5) -```bash -editly subtitle.json5 -``` +## Video overlays with alpha channel + +[alpha.json5](https://github.com/mifi/editly/blob/master/examples/alpha.json5) + +## Image overlays with alpha channel + +[imageOverlay.json5](https://github.com/mifi/editly/blob/master/examples/imageOverlay.json5) + +## Partial overlays (B-roll) + +[visibleFromUntil.json5](https://github.com/mifi/editly/blob/master/examples/visibleFromUntil.json5) + +## Audio layers + +- [audio1.json5](https://github.com/mifi/editly/blob/master/examples/audio1.json5) +- [audio2.json5](https://github.com/mifi/editly/blob/master/examples/audio2.json5) + ## Custom HTML5 canvas Javascript @@ -56,7 +54,6 @@ editly subtitle.json5 [customCanvas.js](https://github.com/mifi/editly/blob/master/examples/customCanvas.js) - ```bash node customCanvas.js ``` @@ -67,11 +64,11 @@ node customCanvas.js [customFabric.js](https://github.com/mifi/editly/blob/master/examples/customFabric.js) - ```bash node customFabric.js ``` + ## LosslessCut tutorial [This video](https://www.youtube.com/watch?v=pYHMxXy05Jg) was created with [losslesscut.json5](https://github.com/mifi/editly/blob/master/examples/losslesscut.json5) \ No newline at end of file From fef9a9e2233b75cb9f52083836da17ea63059dc1 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 21:13:45 +0200 Subject: [PATCH 13/16] add missing example --- examples/imageOverlay.json5 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 examples/imageOverlay.json5 diff --git a/examples/imageOverlay.json5 b/examples/imageOverlay.json5 new file mode 100644 index 0000000..97b608d --- /dev/null +++ b/examples/imageOverlay.json5 @@ -0,0 +1,12 @@ +{ + outPath: './imageOverlay.mp4', + clips: [ + { layers: [ + { type: 'video', path: './assets/IMG_4605.MOV', cutTo: 2 }, + { type: 'image-overlay', path: './assets/overlay.svg', width: 0.2, position: { x: 0.95, y: 0.03, originX: 'right' } }, + { type: 'image-overlay', path: './assets/emoji.png', visibleUntil: 0.5, zoomDirection: 'in' }, + { type: 'image-overlay', path: './assets/emoji.png', position: 'top', visibleFrom: 0.7, visibleUntil: 1.3, width: 0.2 }, + { type: 'image-overlay', path: './assets/emoji.png', position: 'bottom', visibleFrom: 0.7, visibleUntil: 1.3, height: 0.2 }, + ] }, + ], +} From e9647461310070fe8f1c6c7f68e1fcb1d564b1e6 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 21:13:57 +0200 Subject: [PATCH 14/16] fix example --- examples/audio1.json5 | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/audio1.json5 b/examples/audio1.json5 index d38d974..a326774 100644 --- a/examples/audio1.json5 +++ b/examples/audio1.json5 @@ -8,6 +8,7 @@ { duration: 0.5, layers: [{ type: 'video', path: './assets/lofoten.mp4', cutFrom: 0.4, cutTo: 2 }] }, { layers: [ + { type: 'fill-color' }, { type: 'title', text: 'test' }, { type: 'audio', path: './assets/High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a', cutFrom: 2, cutTo: 5 }] }, From 9feb94112f3ad302af3c0bbf12ef7a141b122541 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 21:14:07 +0200 Subject: [PATCH 15/16] improve example --- examples/imageOverlay.json5 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/imageOverlay.json5 b/examples/imageOverlay.json5 index 97b608d..62390f1 100644 --- a/examples/imageOverlay.json5 +++ b/examples/imageOverlay.json5 @@ -5,8 +5,8 @@ { type: 'video', path: './assets/IMG_4605.MOV', cutTo: 2 }, { type: 'image-overlay', path: './assets/overlay.svg', width: 0.2, position: { x: 0.95, y: 0.03, originX: 'right' } }, { type: 'image-overlay', path: './assets/emoji.png', visibleUntil: 0.5, zoomDirection: 'in' }, - { type: 'image-overlay', path: './assets/emoji.png', position: 'top', visibleFrom: 0.7, visibleUntil: 1.3, width: 0.2 }, - { type: 'image-overlay', path: './assets/emoji.png', position: 'bottom', visibleFrom: 0.7, visibleUntil: 1.3, height: 0.2 }, + { type: 'image-overlay', path: './assets/emoji2.svg', position: 'top', visibleFrom: 0.7, visibleUntil: 1.5, width: 0.2 }, + { type: 'image-overlay', path: './assets/emoji2.svg', position: 'bottom', visibleFrom: 0.7, visibleUntil: 1.5, height: 0.2 }, ] }, ], } From 920139ddf360c3842faaa6c61d7c556923cf06af Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 6 Sep 2020 21:14:16 +0200 Subject: [PATCH 16/16] add gif in readme --- examples/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/README.md b/examples/README.md index 32fef3b..e3c03e0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -36,6 +36,8 @@ ## Image overlays with alpha channel +![](https://github.com/mifi/gifs/raw/master/imageOverlay.gif) + [imageOverlay.json5](https://github.com/mifi/editly/blob/master/examples/imageOverlay.json5) ## Partial overlays (B-roll)