From 3aaa999732cd069d9d011ccd983f59b535c6bfe8 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Fri, 17 Apr 2020 00:10:42 +0800 Subject: [PATCH] initial --- .eslintrc | 15 ++ cli.js | 107 ++++++++ colors.js | 199 +++++++++++++++ examples/commonFeatures.json5 | 36 +++ examples/customCanvas.js | 32 +++ examples/customFabric.js | 35 +++ examples/gl.json5 | 16 ++ examples/image.json5 | 7 + examples/losslesscut.json5 | 36 +++ examples/resizeHorizontal.json5 | 13 + examples/resizeVertical.json5 | 14 ++ examples/speedTest.json5 | 15 ++ examples/transitionEasing.json5 | 13 + examples/transparentGradient.json5 | 7 + glTransitions.js | 71 ++++++ index.js | 388 +++++++++++++++++++++++++++++ package.json | 42 ++++ shaders/rainbow-colors.frag | 16 ++ sources/fabricFrameSource.js | 331 ++++++++++++++++++++++++ sources/frameSource.js | 61 +++++ sources/glFrameSource.js | 65 +++++ sources/shared.js | 21 ++ sources/videoFrameSource.js | 124 +++++++++ transitions.js | 71 ++++++ util.js | 34 +++ 25 files changed, 1769 insertions(+) create mode 100644 .eslintrc create mode 100644 cli.js create mode 100644 colors.js create mode 100644 examples/commonFeatures.json5 create mode 100644 examples/customCanvas.js create mode 100644 examples/customFabric.js create mode 100644 examples/gl.json5 create mode 100644 examples/image.json5 create mode 100644 examples/losslesscut.json5 create mode 100644 examples/resizeHorizontal.json5 create mode 100644 examples/resizeVertical.json5 create mode 100644 examples/speedTest.json5 create mode 100644 examples/transitionEasing.json5 create mode 100644 examples/transparentGradient.json5 create mode 100644 glTransitions.js create mode 100644 index.js create mode 100644 package.json create mode 100644 shaders/rainbow-colors.frag create mode 100644 sources/fabricFrameSource.js create mode 100644 sources/frameSource.js create mode 100644 sources/glFrameSource.js create mode 100644 sources/shared.js create mode 100644 sources/videoFrameSource.js create mode 100644 transitions.js create mode 100644 util.js diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..e7858a5 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,15 @@ +{ + "extends": "airbnb-base", + "env": { + "node": true + }, + "parserOptions": { + "sourceType": "script" + }, + "rules": { + "max-len": 0, + "no-console": 0, + "object-curly-newline": 0, + "no-await-in-loop": 0, + } +} diff --git a/cli.js b/cli.js new file mode 100644 index 0000000..272515c --- /dev/null +++ b/cli.js @@ -0,0 +1,107 @@ +#!/usr/bin/env node +const meow = require('meow'); +const fs = require('fs'); +const FileType = require('file-type'); +const pMap = require('p-map'); +const JSON5 = require('json5'); +const assert = require('assert'); + +const editly = require('.'); + + +const cli = meow(` + Usage + $ editly CLIP1 [CLIP2 [CLIP3 ...]] + where each CLIP can be one of the following: + - A path to a video file + - A path to an image + - A quoted text to show in a title screen, prefixed by "title:" + + Or alternatively: + $ editly --json JSON_PATH + where JSON_PATH is the path to an edit spec JSON file, can be a normal JSON or JSON5 + + Options + --out Out video path (defaults to ./editly-out.mp4) - can also be a .gif + --json Use JSON config, all other options will be ignored + --transition-name Name of default transition to use + --transition-duration Default transition duration in milliseconds + --width Width which all videos will be converted to + --height Height which all videos will be converted to + --fps FPS which all videos will be converted to + --font-path Set default font to a .ttf + --audio-file-path Add an audio track + + --fast, -f Fast mode (low resolution and FPS, useful for getting a quick preview) + --verbose + + Examples + $ editly title:'My video' clip1.mov clip2.mov title:'My slideshow' img1.jpg img2.jpg title:'THE END' --audio-file-path /path/to/music.mp3 --font-path /path/to/my-favorite-font.ttf + $ editly --json my-editly.json --out output.gif +`, { + flags: { + fast: { type: 'boolean', alias: 'f' }, + transitionDuration: { type: 'number' }, + width: { type: 'number' }, + height: { type: 'number' }, + fps: { type: 'number' }, + }, +}); + +(async () => { + let params = { + defaults: {}, + }; + if (cli.flags.json) { + params = JSON5.parse(fs.readFileSync(cli.flags.json, 'utf-8')); + } else { + const clipsIn = cli.input; + if (clipsIn.length < 1) cli.showHelp(); + + const clips = await pMap(clipsIn, async (clip) => { + const match = clip.match(/^title:(.+)$/); + if (match) return { type: 'title-background', text: match[1] }; + + const { mime } = await FileType.fromFile(clip); + + if (mime.startsWith('video')) return { type: 'video', path: clip }; + if (mime.startsWith('image')) return { type: 'image', path: clip }; + + throw new Error(`Unrecognized clip or file type "${clip}"`); + }, { concurrency: 1 }); + + assert(clips.length > 0, 'No clips specified'); + + params.clips = clips.map((clip) => ({ layers: [clip] })); + } + + const { verbose, transitionName, transitionDuration, width, height, fps, audioFilePath, fontPath, fast, out: outPath } = cli.flags; + + if (transitionName || transitionDuration) { + params.defaults.transition = { + name: transitionName, + duration: transitionDuration, + }; + } + + if (fontPath) { + params.defaults.layer = { + fontPath, + }; + } + + if (outPath) params.outPath = outPath; + if (audioFilePath) params.audioFilePath = audioFilePath; + if (width) params.width = width; + if (height) params.height = height; + if (fps) params.fps = fps; + + if (fast) params.fast = fast; + if (verbose) params.verbose = verbose; + + if (params.verbose) console.log(JSON5.stringify(params, null, 2)); + + if (!params.outPath) params.outPath = './editly-out.mp4'; + + await editly(params); +})().catch(console.error); diff --git a/colors.js b/colors.js new file mode 100644 index 0000000..b171f41 --- /dev/null +++ b/colors.js @@ -0,0 +1,199 @@ +// TODO make separate npm module + +// https://stackoverflow.com/a/4382138/6519037 +const allColors = [ + 'hsl(42, 100%, 50%)', + 'hsl(310, 34%, 37%)', + 'hsl(24, 100%, 50%)', + 'hsl(211, 38%, 74%)', + 'hsl(350, 100%, 37%)', + 'hsl(35, 52%, 59%)', + 'hsl(22, 11%, 45%)', + 'hsl(145, 100%, 24%)', + 'hsl(348, 87%, 71%)', + 'hsl(203, 100%, 27%)', + 'hsl(11, 100%, 68%)', + 'hsl(265, 37%, 34%)', + 'hsl(33, 100%, 50%)', + 'hsl(342, 63%, 42%)', + 'hsl(49, 100%, 47%)', + 'hsl(5, 81%, 27%)', + 'hsl(68, 100%, 33%)', + 'hsl(26, 61%, 21%)', + 'hsl(10, 88%, 51%)', + 'hsl(84, 33%, 12%)', +]; + +// https://digitalsynopsis.com/design/beautiful-color-ui-gradients-backgrounds/ +const gradientColors = [ + [ + '#ff9aac', + '#ffa875', + ], + [ + '#cc2b5e', + '#753a88', + ], + [ + '#42275a', + '#734b6d', + ], + [ + '#bdc3c7', + '#2c3e50', + ], + [ + '#de6262', + '#ffb88c', + ], + [ + '#eb3349', + '#f45c43', + ], + [ + '#dd5e89', + '#f7bb97', + ], + [ + '#56ab2f', + '#a8e063', + ], + [ + '#614385', + '#516395', + ], + [ + '#eecda3', + '#ef629f', + ], + [ + '#eacda3', + '#d6ae7b', + ], + [ + '#02aab0', + '#00cdac', + ], + [ + '#d66d75', + '#e29587', + ], + [ + '#000428', + '#004e92', + ], + [ + '#ddd6f3', + '#faaca8', + ], + [ + '#7b4397', + '#dc2430', + ], + [ + '#43cea2', + '#185a9d', + ], + [ + '#ba5370', + '#f4e2d8', + ], + [ + '#ff512f', + '#dd2476', + ], + [ + '#4568dc', + '#b06ab3', + ], + [ + '#ec6f66', + '#f3a183', + ], + [ + '#ffd89b', + '#19547b', + ], + [ + '#3a1c71', + '#d76d77', + ], + [ + '#4ca1af', + '#c4e0e5', + ], + [ + '#ff5f6d', + '#ffc371', + ], + [ + '#36d1dc', + '#5b86e5', + ], + [ + '#c33764', + '#1d2671', + ], + [ + '#141e30', + '#243b55', + ], + [ + '#ff7e5f', + '#feb47b', + ], + [ + '#ed4264', + '#ffedbc', + ], + [ + '#2b5876', + '#4e4376', + ], + [ + '#ff9966', + '#ff5e62', + ], + [ + '#aa076b', + '#61045f', + ], +]; + +/* const lightGradients = [ + [ + '#ee9ca7', + '#ffdde1', + ], + [ + '#2193b0', + '#6dd5ed', + ], +]; */ + +function getRandomColor(colors = allColors) { + const index = Math.floor(Math.random() * colors.length); + const remainingColors = [...colors]; + remainingColors.splice(index, 1); + return { remainingColors, color: colors[index] || allColors[0] }; +} + +function getRandomColors(num) { + let colors = allColors; + const out = []; + for (let i = 0; i < Math.min(num, allColors.length); i += 1) { + const { remainingColors, color } = getRandomColor(colors); + out.push(color); + colors = remainingColors; + } + return out; +} + +function getRandomGradient() { + return gradientColors[Math.floor(Math.random() * gradientColors.length)]; +} + +module.exports = { + getRandomColors, + getRandomGradient, +}; diff --git a/examples/commonFeatures.json5 b/examples/commonFeatures.json5 new file mode 100644 index 0000000..ce98c45 --- /dev/null +++ b/examples/commonFeatures.json5 @@ -0,0 +1,36 @@ +{ + // fast: true, + // width: 2166, height: 1650, fps: 30, + outPath: './out.mp4', + // outPath: './out.gif', + // verbose: true, + // enableFfmpegLog: true, + audioFilePath: './High [NCS Release] - JPB (No Copyright Music)-R8ZRCXy5vhA.m4a', + defaults: { + transition: { name: 'random' }, + layer: { fontPath: './Patua_One/PatuaOne-Regular.ttf' }, + }, + clips: [ + { duration: 3, transition: { name: 'directional-left' }, layers: [{ type: 'title-background', text: 'EDITLY\nVideo editing framework', background: { type: 'linear-gradient', colors: ['#02aab0', '#00cdac'] } }] }, + { duration: 4, transition: { name: 'dreamyzoom' }, layers: [{ type: 'title-background', text: 'Multi-line text with animated linear or radial gradients', background: { type: 'radial-gradient' } }] }, + { duration: 3, transition: { name: 'directional-right' }, layers: [{ type: 'rainbow-colors' }, { type: 'title', text: 'Colorful backgrounds' }] }, + { duration: 3, layers: [{ type: 'pause' }, { type: 'title', text: 'and separators' }] }, + + { duration: 3, transition: { name: 'fadegrayscale' }, layers: [{ type: 'title-background', text: 'Image slideshows with Ken Burns effect', background: { type: 'linear-gradient' } }] }, + { duration: 2.5, transition: { name: 'directionalWarp' }, layers: [{ type: 'image', path: './vertical.jpg', zoomDirection: 'out' }] }, + { duration: 3, transition: { name: 'dreamyzoom' }, layers: [{ type: 'image', path: './img1.jpg', duration: 2.5, zoomDirection: 'in' }, { type: 'subtitle', text: 'Indonesia has many spectacular locations. Here is the volcano Kelimutu, which has three lakes in its core, some days with three different colors!' }, { type: 'title', position: 'top', text: 'With text' }] }, + { duration: 3, transition: { name: 'colorphase' }, layers: [{ type: 'image', path: './img2.jpg', zoomDirection: 'out' }, { type: 'subtitle', text: 'Komodo national park is the only home of the endangered Komodo dragons' }] }, + { duration: 2.5, transition: { name: 'simplezoom' }, layers: [{ type: 'image', path: './img3.jpg', zoomDirection: 'in' }] }, + + { duration: 1.5, transition: { name: 'crosszoom', duration: 0.3 }, layers: [{ type: 'video', path: '/Users/mifi/Desktop/photos/drone koh lipe/DJI_0402.MOV', cutTo: 58 }, { type: 'title', text: 'Videos' }] }, + { duration: 3, transition: { name: 'fade' }, layers: [{ type: 'video', path: '/Users/mifi/Desktop/photos/drone koh lipe/DJI_0402.MOV', cutFrom: 58 }] }, + { transition: { name: 'fade' }, layers: [{ type: 'video', path: '/Users/mifi/Desktop/photos/drone koh lipe/DJI_0403.MOV', cutTo: 2.5 }] }, + { duration: 1.5, layers: [{ type: 'video', path: '/Users/mifi/Desktop/photos/drone koh lipe/DJI_0401.MOV', cutFrom: 3, cutTo: 30 }] }, + + { duration: 3, transition: { name: 'crosszoom' }, layers: [{ type: 'gl', fragmentPath: './shaders/3l23Rh.frag' }, { type: 'title', text: 'OpenGL\nshaders' }] }, + { duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/MdXyzX.frag' }] }, + { duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/30daysofshade_010.frag' }] }, + { duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/wd2yDm.frag', speed: 5 }] }, + { duration: 3, layers: [{ type: 'editly-banner' }] }, + ], +} diff --git a/examples/customCanvas.js b/examples/customCanvas.js new file mode 100644 index 0000000..85c1c08 --- /dev/null +++ b/examples/customCanvas.js @@ -0,0 +1,32 @@ +const editly = require('..'); + +async function func({ width, height, canvas }) { + async function onRender(progress) { + const context = canvas.getContext('2d'); + const centerX = canvas.width / 2; + const centerY = canvas.height / 2; + const radius = 40 * (1 + progress * 0.5); + + context.beginPath(); + context.arc(centerX, centerY, radius, 0, 2 * Math.PI, false); + context.fillStyle = 'hsl(350, 100%, 37%)'; + context.fill(); + context.lineWidth = 5; + context.strokeStyle = '#ffffff'; + context.stroke(); + } + + function onClose() { + // Cleanup if you initialized anything + } + + return { onRender, onClose }; +} + +editly({ + fast: true, + outPath: './canvas.mp4', + clips: [ + { duration: 2, layers: [{ type: 'canvas', func }] }, + ], +}).catch(console.error); diff --git a/examples/customFabric.js b/examples/customFabric.js new file mode 100644 index 0000000..cd3b300 --- /dev/null +++ b/examples/customFabric.js @@ -0,0 +1,35 @@ +const editly = require('..'); + +/* eslint-disable spaced-comment,no-param-reassign */ + +async function func({ width, height, fabric, canvas }) { + async function onRender(progress) { + canvas.backgroundColor = 'hsl(33, 100%, 50%)'; + + const text = new fabric.Text(`PROGRESS\n${Math.floor(progress * 100)}%`, { + originX: 'center', + originY: 'center', + left: width / 2, + top: (height / 2) * (1 + (progress * 0.1 - 0.05)), + fontSize: 20, + textAlign: 'center', + fill: 'white', + }); + + canvas.add(text); + } + + function onClose() { + // Cleanup if you initialized anything + } + + return { onRender, onClose }; +} + +editly({ + fast: true, + outPath: './fabric.mp4', + clips: [ + { duration: 2, layers: [{ type: 'fabric', func }] }, + ], +}).catch(console.error); diff --git a/examples/gl.json5 b/examples/gl.json5 new file mode 100644 index 0000000..3d13a44 --- /dev/null +++ b/examples/gl.json5 @@ -0,0 +1,16 @@ +{ + fast: true, + // verbose: true, + outPath: './gl.mp4', + defaults: { + transition: { name: 'random' }, + layer: { fontPath: './Patua_One/PatuaOne-Regular.ttf' }, + }, + clips: [ + { transition: null, duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/3l23Rh.frag' }] }, + { duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/MdXyzX.frag' }] }, + { duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/30daysofshade_010.frag', speed: 1 }] }, + { duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/rainbow-background.frag' }] }, + { duration: 3, layers: [{ type: 'gl', fragmentPath: './shaders/wd2yDm.frag', speed: 5 }] }, + ], +} \ No newline at end of file diff --git a/examples/image.json5 b/examples/image.json5 new file mode 100644 index 0000000..d38bf23 --- /dev/null +++ b/examples/image.json5 @@ -0,0 +1,7 @@ +{ + fast: true, + outPath: './image.mp4', + clips: [ + { layers: [{ type: 'image', path: './vertical.jpg', zoomDirection: 'out' }] }, + ], +} \ No newline at end of file diff --git a/examples/losslesscut.json5 b/examples/losslesscut.json5 new file mode 100644 index 0000000..0adc381 --- /dev/null +++ b/examples/losslesscut.json5 @@ -0,0 +1,36 @@ +{ + // fast: false, + outPath: './losslesscut.mp4', + // verbose: true, + // enableFfmpegLog: true, + fps: 30, + audioFilePath: './Believe - Roa [Vlog No Copyright Music]-qldyHxWPFUY.m4a', + defaults: { + transition: { name: 'crossZoom', duration: 1 }, + layer: { fontPath: './Patua_One/PatuaOne-Regular.ttf' }, + }, + clips: [ + { duration: 3, layers: [{ type: 'title-background', text: 'LosslessCut', background: { type: 'linear-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/intro.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Capture full resolution screenshots', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/capture screenshots.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Extract tracks as individual files', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/extract tracks as individual files.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Keyframes and zoom', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/keyframes and zoom.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Label segments', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/label segments.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Lossless rotation', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/lossless rotation.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Thumbnails', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/thumbnails.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Audio waveforms', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/audio waveform.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Track information', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/track information.mov' }] }, + { duration: 3, layers: [{ type: 'title-background', text: 'Tracks editor and audio swap', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: '/Users/mifi/Desktop/losslesscut-usage/tracks editor and replace audio.mov' }] }, + { duration: 4, layers: [{ type: 'title-background', text: 'Get it from\nMac App Store\nWindows Store', background: { type: 'color', color: 'black' } }] }, + { duration: 2, layers: [{ type: 'editly-banner' }] }, + ], +} \ No newline at end of file diff --git a/examples/resizeHorizontal.json5 b/examples/resizeHorizontal.json5 new file mode 100644 index 0000000..77ae2a6 --- /dev/null +++ b/examples/resizeHorizontal.json5 @@ -0,0 +1,13 @@ +{ + fast: true, + outPath: './resizeHorizontal.mp4', + defaults: { + transition: { duration: 0 }, + layer: { fontPath: './Patua_One/PatuaOne-Regular.ttf', backgroundColor: 'white' }, + }, + clips: [ + { layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0.4, cutTo: 2 }] }, + { layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'contain' }] }, + { layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0.4, cutTo: 2, resizeMode: 'stretch' }] }, + ], +} diff --git a/examples/resizeVertical.json5 b/examples/resizeVertical.json5 new file mode 100644 index 0000000..4f1ce43 --- /dev/null +++ b/examples/resizeVertical.json5 @@ -0,0 +1,14 @@ +{ + width: 240, height: 320, fps: 15, + outPath: './resizeVertical.mp4', + defaults: { + transition: { duration: 0 }, + layer: { fontPath: './Patua_One/PatuaOne-Regular.ttf' }, + }, + clips: [ + { duration: 2, layers: [{ type: 'title-background', text: 'Editly can handle all formats and sizes with different fits', background: { type: 'radial-gradient' } }] }, + { layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'contain' }, { type: 'title-background', text: 'Contain' }] }, + { layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0, cutTo: 2, resizeMode: 'stretch' }, { type: 'title-background', text: 'Stretch' }] }, + { layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title-background', text: 'Cover' }] }, + ], +} \ No newline at end of file diff --git a/examples/speedTest.json5 b/examples/speedTest.json5 new file mode 100644 index 0000000..5efc51c --- /dev/null +++ b/examples/speedTest.json5 @@ -0,0 +1,15 @@ +{ + fast: true, + // verbose: true, + outPath: './speedTest.mp4', + defaults: { + transition: null, + layer: { fontPath: './Patua_One/PatuaOne-Regular.ttf' }, + }, + clips: [ + { duration: 2, layers: [{ type: 'title-background', text: 'Speed up or slow down video', background: { type: 'radial-gradient' } }] }, + { duration: 2, layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0, cutTo: 2 }, { type: 'title-background', text: 'Same speed' }] }, + { duration: 1, layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0, cutTo: 4 }, { type: 'title-background', text: '4x' }] }, + { duration: 2, layers: [{ type: 'video', path: './IMG_4605.MOV', cutFrom: 0, cutTo: 1 }, { type: 'title-background', text: '1/2x' }] }, + ], +} \ No newline at end of file diff --git a/examples/transitionEasing.json5 b/examples/transitionEasing.json5 new file mode 100644 index 0000000..f0d1e90 --- /dev/null +++ b/examples/transitionEasing.json5 @@ -0,0 +1,13 @@ +{ + fast: true, + outPath: './transitionEasing.mp4', + defaults: { + duration: 2, + }, + clips: [ + { transition: { name: 'directional', duration: 0.5 }, layers: [{ type: 'video', path: '/Users/mifi/Desktop/photos/drone koh lipe/DJI_0402.MOV', cutTo: 2 }] }, + { transition: { name: 'directional', duration: 0.5, params: { direction: [1, 0] } }, layers: [{ type: 'video', path: '/Users/mifi/Desktop/photos/drone koh lipe/DJI_0403.MOV', cutTo: 2 }] }, + // { transition: { name: 'directional', duration: 0.5, easing: null }, layers: [{ type: 'video', path: '/Users/mifi/Desktop/photos/drone koh lipe/DJI_0403.MOV', cutTo: 2 }] }, + { layers: [{ type: 'pause' }] }, + ], +} diff --git a/examples/transparentGradient.json5 b/examples/transparentGradient.json5 new file mode 100644 index 0000000..e1c9cb9 --- /dev/null +++ b/examples/transparentGradient.json5 @@ -0,0 +1,7 @@ +{ + // fast: true, + outPath: './transparentGradient.mp4', + clips: [ + { duration: 0.1, layers: [{ type: 'fill-color', color: 'green' }, { type: 'linear-gradient', colors: ['#ffffffff', '#ffffff00'] }] }, + ], +} diff --git a/glTransitions.js b/glTransitions.js new file mode 100644 index 0000000..26b3fe7 --- /dev/null +++ b/glTransitions.js @@ -0,0 +1,71 @@ +const GL = require('gl'); +const ndarray = require('ndarray'); +const createBuffer = require('gl-buffer'); +const transitions = require('gl-transitions'); +const createTransition = require('gl-transition').default; +const createTexture = require('gl-texture2d'); + +module.exports = ({ width, height, channels }) => { + const gl = GL(width, height); + + function runTransitionOnFrame({ fromFrame, toFrame, progress, transitionName, transitionParams = {} }) { + function convertFrame(buf) { + // @see https://github.com/stackgl/gl-texture2d/issues/16 + return ndarray(buf, [width, height, channels], [channels, width * channels, 1]); + } + + const buffer = createBuffer(gl, + [-1, -1, -1, 4, 4, -1], + gl.ARRAY_BUFFER, + gl.STATIC_DRAW); + + let transition; + + try { + const resizeMode = 'stretch'; + + const transitionSource = transitions.find((t) => t.name.toLowerCase() === transitionName.toLowerCase()); + + transition = createTransition(gl, transitionSource, { resizeMode }); + + gl.clear(gl.COLOR_BUFFER_BIT); + + // console.time('runTransitionOnFrame internal'); + const fromFrameNdArray = convertFrame(fromFrame); + const textureFrom = createTexture(gl, fromFrameNdArray); + textureFrom.minFilter = gl.LINEAR; + textureFrom.magFilter = gl.LINEAR; + + // console.timeLog('runTransitionOnFrame internal'); + const toFrameNdArray = convertFrame(toFrame); + const textureTo = createTexture(gl, toFrameNdArray); + textureTo.minFilter = gl.LINEAR; + textureTo.magFilter = gl.LINEAR; + + buffer.bind(); + transition.draw(progress, textureFrom, textureTo, gl.drawingBufferWidth, gl.drawingBufferHeight, transitionParams); + + textureFrom.dispose(); + textureTo.dispose(); + + // console.timeLog('runTransitionOnFrame internal'); + + const outArray = Buffer.allocUnsafe(width * height * 4); + gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, outArray); + + // console.timeEnd('runTransitionOnFrame internal'); + + return outArray; + + // require('fs').writeFileSync(`${new Date().getTime()}.raw`, outArray); + // Testing: ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i 1586619627191.raw -vf format=yuv420p -vcodec libx264 -y out.mp4 + } finally { + buffer.dispose(); + if (transition) transition.dispose(); + } + } + + return { + runTransitionOnFrame, + }; +}; diff --git a/index.js b/index.js new file mode 100644 index 0000000..faf97dd --- /dev/null +++ b/index.js @@ -0,0 +1,388 @@ +const execa = require('execa'); +const assert = require('assert'); +const pMap = require('p-map'); +const { basename, join } = require('path'); +const flatMap = require('lodash/flatMap'); +const JSON5 = require('json5'); + +const { parseFps, readFileInfo, multipleOf2 } = require('./util'); +const { registerFont } = require('./sources/fabricFrameSource'); +const { createFrameSource } = require('./sources/frameSource'); +const { calcTransition } = require('./transitions'); + +const GlTransitions = require('./glTransitions'); + +// Cache +const loadedFonts = []; + + +module.exports = async (config = {}) => { + const { + // Testing options: + enableFfmpegLog = false, + verbose = false, + fast, + + outPath, + clips: clipsIn, + width: requestedWidth, + height: requestedHeight, + fps: requestedFps, + defaults: defaultsIn = {}, + audioFilePath: audioFilePathIn, + } = config; + + const isGif = outPath.toLowerCase().endsWith('.gif'); + + const audioFilePath = isGif ? undefined : audioFilePathIn; + + const defaults = { + duration: 4, + ...defaultsIn, + transition: defaultsIn.transition === null ? null : { + duration: 0.5, + name: 'random', + ...defaultsIn.transition, + }, + }; + + if (verbose) console.log(JSON5.stringify(config, null, 2)); + + assert(outPath, 'Please provide an output path'); + assert(clipsIn.length > 0, 'Please provide at least 1 clip'); + + async function handleLayer(layer) { + const { type, ...restLayer } = layer; + + if (['fabric', 'canvas'].includes(type)) assert(typeof layer.func === 'function', '"func" must be a function'); + + if (['image', 'fabric', 'canvas', 'gl', 'radial-gradient', 'linear-gradient', 'fill-color'].includes(type)) return layer; + + // TODO if random-background radial-gradient linear etc + if (type === 'pause') return handleLayer({ ...restLayer, type: 'fill-color' }); + + if (type === 'rainbow-colors') return handleLayer({ type: 'gl', fragmentPath: join(__dirname, 'shaders/rainbow-colors.frag') }); + + if (type === 'editly-banner') { + const { fontPath } = layer; + return [ + await handleLayer({ type: 'linear-gradient' }), + await handleLayer({ fontPath, type: 'title', text: 'Made with\nEDITLY\nmifi.no' }), + ]; + } + + // For convenience + if (type === 'title-background') { + const { text, textColor, background, fontFamily, fontPath } = layer; + const outLayers = []; + if (background) { + if (background.type === 'radial-gradient') outLayers.push(await handleLayer({ type: 'radial-gradient', colors: background.colors })); + else if (background.type === 'linear-gradient') outLayers.push(await handleLayer({ type: 'linear-gradient', colors: background.colors })); + else if (background.color) outLayers.push(await handleLayer({ type: 'fill-color', color: background.color })); + } else { + const backgroundTypes = ['radial-gradient', 'linear-gradient', 'fill-color']; + const randomType = backgroundTypes[Math.floor(Math.random() * backgroundTypes.length)]; + outLayers.push(await handleLayer({ type: randomType })); + } + outLayers.push(await handleLayer({ type: 'title', fontFamily, fontPath, text, textColor })); + return outLayers; + } + + if (type === 'title' || type === 'subtitle') { + assert(layer.text, 'Please specify a text'); + + let { fontFamily } = layer; + const { fontPath, ...rest } = layer; + if (fontPath) { + fontFamily = Buffer.from(basename(fontPath)).toString('base64'); + if (!loadedFonts.includes(fontFamily)) { + registerFont(fontPath, { family: fontFamily, weight: 'regular', style: 'normal' }); + loadedFonts.push(fontFamily); + } + } + return { ...rest, fontFamily }; + } + + throw new Error(`Invalid layer type ${type}`); + } + + const clips = await pMap(clipsIn, async (clip, clipIndex) => { + const { transition: userTransition, duration: userDuration, layers } = { ...clip }; + + const videoLayers = layers.filter((layer) => layer.type === 'video'); + assert(videoLayers.length <= 1, 'Max 1 video per layer'); + + const userOrDefaultDuration = userDuration || defaults.duration; + if (videoLayers.length === 0) assert(userOrDefaultDuration, `Duration is required for clip ${clipIndex}`); + + let duration = userOrDefaultDuration; + + const layersOut = flatMap(await pMap(layers, async (layerIn) => { + const layer = { ...defaults.layer, ...layerIn }; + const { type } = layer; + + if (type === 'video') { + const { cutFrom: cutFromIn, cutTo: cutToIn, path } = layer; + const fileInfo = await readFileInfo(path); + const { duration: fileDuration, width, height, framerateStr } = fileInfo; + let cutFrom; + let cutTo; + let trimmedSourceDuration = fileDuration; + if (cutFromIn != null || cutToIn != null) { + cutFrom = Math.min(Math.max(0, cutFromIn || 0), fileDuration); + cutTo = Math.min(Math.max(cutFrom, cutToIn || fileDuration), fileDuration); + assert(cutFrom < cutTo, 'cutFrom must be lower than cutTo'); + + trimmedSourceDuration = cutTo - cutFrom; + } + + // If user specified duration, means that should be the output duration + let framePtsFactor; + if (userDuration) { + duration = userDuration; + framePtsFactor = userDuration / trimmedSourceDuration; + } else { + duration = trimmedSourceDuration; + framePtsFactor = 1; + } + + return { ...layer, cutFrom, cutTo, width, height, framerateStr, framePtsFactor }; + } + + return handleLayer(layer); + }, { concurrency: 1 })); + + const transition = calcTransition(defaults, userTransition); + + return { + transition, + duration, + layers: layersOut, + }; + }, { concurrency: 1 }); + + if (verbose) console.log(JSON5.stringify(clips, null, 2)); + + // Try to detect parameters from first video + let detectedWidth; + let detectedHeight; + let firstVideoFramerateStr; + + clips.find((clip) => clip && clip.layers.find((layer) => { + if (layer.type === 'video') { + detectedWidth = layer.width; + detectedHeight = layer.height; + firstVideoFramerateStr = layer.framerateStr; + + return true; + } + return false; + })); + + let width; + let height; + + let desiredWidth; + if (requestedWidth) desiredWidth = requestedWidth; + else if (fast) desiredWidth = 320; + else if (isGif) desiredWidth = 320; + else desiredWidth = 640; + + if (detectedWidth && detectedHeight) { + const calculatedHeight = Math.round((detectedHeight / detectedWidth) * desiredWidth); + height = isGif ? calculatedHeight : multipleOf2(calculatedHeight); // x264 requires multiple of 2 + width = desiredWidth; + } else { + // Cannot detect width/height from video, set defaults + width = desiredWidth; + height = desiredWidth; + } + + // User override? + if (requestedWidth && requestedHeight) { + width = requestedWidth; + height = requestedHeight; + } + + assert(width, 'Width not specified or detected'); + assert(height, 'Height not specified or detected'); + + let fps; + let framerateStr; + + if (requestedFps && typeof requestedFps === 'number') { + fps = requestedFps; + framerateStr = String(requestedFps); + } else if (fast) { + fps = 15; + framerateStr = String(fps); + } else if (isGif) { + fps = 10; + framerateStr = String(fps); + } else if (firstVideoFramerateStr) { + fps = parseFps(firstVideoFramerateStr); + framerateStr = firstVideoFramerateStr; + } else { + fps = 25; + framerateStr = String(fps); + } + + assert(fps, 'FPS not specified or detected'); + + console.log(`${width}x${height} ${fps}fps`); + + const channels = 4; + + const { runTransitionOnFrame } = GlTransitions({ width, height, channels }); + + function startFfmpegWriterProcess() { + // https://superuser.com/questions/556029/how-do-i-convert-a-video-to-gif-using-ffmpeg-with-reasonable-quality + const outputArgs = isGif ? [ + '-vf', + `fps=${fps},scale=${width}:${height}:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse`, + '-loop', 0, + '-y', outPath, + ] : [ + '-vf', 'format=yuv420p', + '-vcodec', 'libx264', + '-profile:v', 'high', + ...(fast ? ['-preset:v', 'ultrafast'] : ['-preset:v', 'medium']), + '-crf', '18', + + '-movflags', 'faststart', + '-y', outPath, + ]; + const args = [ + ...(enableFfmpegLog ? [] : ['-hide_banner', '-loglevel', 'panic']), + + '-f', 'rawvideo', + '-vcodec', 'rawvideo', + '-pix_fmt', 'rgba', + '-s', `${width}x${height}`, + '-r', framerateStr, + '-i', '-', + + ...(audioFilePath ? ['-i', audioFilePath, '-shortest'] : []), + ...(audioFilePath ? ['-acodec', 'aac', '-b:a', '128k'] : []), + + ...outputArgs, + ]; + if (verbose) console.log('ffmpeg', args.join(' ')); + return execa('ffmpeg', args, { encoding: null, buffer: false, stdin: 'pipe', stdout: process.stdout, stderr: process.stderr }); + } + + let outProcess; + let frameSource1; + let frameSource2; + + try { + outProcess = startFfmpegWriterProcess(); + + let totalFrameCount = 0; + let fromClipFrameCount = 0; + let toClipFrameCount = 0; + + let transitionFromClipId = 0; + + const getTransitionToClipId = () => transitionFromClipId + 1; + const getTransitionFromClip = () => clips[transitionFromClipId]; + const getTransitionToClip = () => clips[getTransitionToClipId()]; + + const getSource = (clip, clipIndex) => createFrameSource({ clip, clipIndex, width, height, channels, verbose, enableFfmpegLog, framerateStr }); + + const getTransitionToSource = async () => (getTransitionToClip() && getSource(getTransitionToClip(), getTransitionToClipId())); + frameSource1 = await getSource(getTransitionFromClip(), transitionFromClipId); + frameSource2 = await getTransitionToSource(); + + // eslint-disable-next-line no-constant-condition + while (true) { + const fromClipNumFrames = Math.round(getTransitionFromClip().duration * fps); + const toClipNumFrames = getTransitionToClip() && Math.round(getTransitionToClip().duration * fps); + const fromClipProgress = fromClipFrameCount / fromClipNumFrames; + const toClipProgress = getTransitionToClip() && toClipFrameCount / toClipNumFrames; + const frameData1 = await frameSource1.readNextFrame(fromClipProgress); + + const clipTransition = getTransitionFromClip().transition; + + const transitionNumFrames = Math.round((clipTransition.duration || 0) * fps); + + // Each clip has two transitions, make sure we leave enough room: + const transitionNumFramesSafe = Math.floor(Math.min(Math.min(fromClipNumFrames, toClipNumFrames != null ? toClipNumFrames : Number.MAX_SAFE_INTEGER) / 2, transitionNumFrames)); + // How many frames into the transition are we? negative means not yet started + const transitionFrameAt = fromClipFrameCount - (fromClipNumFrames - transitionNumFramesSafe); + + if (verbose) console.log('Frame', totalFrameCount, 'from', fromClipFrameCount, `(clip ${transitionFromClipId})`, 'to', toClipFrameCount, `(clip ${getTransitionToClipId()})`); + + if (!frameData1 || transitionFrameAt >= transitionNumFramesSafe - 1) { + // if (!frameData1 || transitionFrameAt >= transitionNumFramesSafe) { + console.log('Done with transition, switching to next clip'); + transitionFromClipId += 1; + + if (!getTransitionFromClip()) { + console.log('No more transitionFromClip, done'); + break; + } + + // Cleanup old, swap and load next + await frameSource1.close(); + frameSource1 = frameSource2; + frameSource2 = await getTransitionToSource(); + + fromClipFrameCount = transitionNumFramesSafe; + toClipFrameCount = 0; + } else { + let outFrameData; + if (frameSource2 && transitionFrameAt >= 0) { + if (verbose) console.log('Transition', 'frame', transitionFrameAt, '/', transitionNumFramesSafe, clipTransition.name, `${clipTransition.duration}s`); + + const frameData2 = await frameSource2.readNextFrame(toClipProgress); + toClipFrameCount += 1; + + if (frameData2) { + const progress = transitionFrameAt / transitionNumFramesSafe; + const easedProgress = clipTransition.easingFunction(progress); + + if (verbose) console.time('runTransitionOnFrame'); + outFrameData = runTransitionOnFrame({ fromFrame: frameData1, toFrame: frameData2, progress: easedProgress, transitionName: clipTransition.name, transitionParams: clipTransition.params }); + if (verbose) console.timeEnd('runTransitionOnFrame'); + } else { + console.warn('Got no frame data from clip 2!'); + // We have reached end of clip2 but transition is not complete + // Pass thru + // TODO improve, maybe cut it short + outFrameData = frameData1; + } + } else { + outFrameData = frameData1; + } + + // If we don't await we get EINVAL when dealing with high resolution files (big writes) + await new Promise((r) => outProcess.stdin.write(outFrameData, () => r())); + // outProcess.stdin.write(outFrameData); + + fromClipFrameCount += 1; + } + + totalFrameCount += 1; + } + + outProcess.stdin.end(); + + console.log('Done. Output file can be found at:'); + console.log(outPath); + } catch (err) { + console.error('Loop failed', err); + if (outProcess) { + outProcess.kill(); + } + } finally { + if (frameSource1) await frameSource1.close(); + if (frameSource2) await frameSource2.close(); + } + + try { + await outProcess; + } catch (err) { + if (!err.killed) throw err; + } +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..7a8b76a --- /dev/null +++ b/package.json @@ -0,0 +1,42 @@ +{ + "name": "editly", + "description": "Simple, sexy, declarative video editing", + "version": "0.1.0", + "main": "index.js", + "author": "Mikael Finstad ", + "license": "MIT", + "dependencies": { + "canvas": "^2.6.1", + "execa": "^4.0.0", + "fabric": "^3.6.3", + "file-type": "^14.1.4", + "file-url": "^3.0.0", + "fs-extra": "^9.0.0", + "gl": "^4.5.0", + "gl-buffer": "^2.1.2", + "gl-shader": "^4.2.1", + "gl-texture2d": "^2.1.0", + "gl-transition": "^1.13.0", + "gl-transitions": "^1.43.0", + "json5": "^2.1.3", + "lodash": "^4.17.15", + "meow": "^6.1.0", + "ndarray": "^1.0.19", + "p-map": "^4.0.0" + }, + "scripts": { + "test": "exit 0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/mifi/editly.git" + }, + "bin": { + "editly": "cli.js" + }, + "devDependencies": { + "eslint": "^5.16.0 || ^6.8.0", + "eslint-config-airbnb-base": "^14.1.0", + "eslint-plugin-import": "^2.20.1" + } +} diff --git a/shaders/rainbow-colors.frag b/shaders/rainbow-colors.frag new file mode 100644 index 0000000..a4be5aa --- /dev/null +++ b/shaders/rainbow-colors.frag @@ -0,0 +1,16 @@ +#ifdef GL_ES +precision mediump float; +#endif + +uniform float time; +uniform vec2 resolution; + +void main() { + vec2 st = gl_FragCoord.xy/resolution.xy; + st.x *= resolution.x/resolution.y; + + vec3 color = vec3(0.); + color = vec3(st.x,st.y,abs(sin(time))); + + gl_FragColor = vec4(color,1.0); +} diff --git a/sources/fabricFrameSource.js b/sources/fabricFrameSource.js new file mode 100644 index 0000000..74b8299 --- /dev/null +++ b/sources/fabricFrameSource.js @@ -0,0 +1,331 @@ +const { fabric } = require('fabric'); +const fileUrl = require('file-url'); +const nodeCanvas = require('canvas'); + +const { createCanvas } = nodeCanvas; + +const { canvasToRgba } = require('./shared'); +const { getRandomGradient, getRandomColors } = require('../colors'); +const { easeOutExpo } = require('../transitions'); + +// http://fabricjs.com/kitchensink + + +function fabricCanvasToRgba(canvas) { + // https://github.com/fabricjs/fabric.js/blob/26e1a5b55cbeeffb59845337ced3f3f91d533d7d/src/static_canvas.class.js + // https://github.com/fabricjs/fabric.js/issues/3885 + const internalCanvas = fabric.util.getNodeCanvas(canvas.lowerCanvasEl); + const ctx = internalCanvas.getContext('2d'); + + // require('fs').writeFileSync(`${Math.floor(Math.random() * 1e12)}.png`, internalCanvas.toBuffer('image/png')); + // throw new Error('abort'); + + return canvasToRgba(ctx); +} + +async function mergeFrames({ width, height, framesRaw }) { + if (framesRaw.length === 1) return framesRaw[0]; + + // Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668 + const canvas = createCanvas(width, height); + const ctx = canvas.getContext('2d'); + + framesRaw.forEach((frameRaw) => { + const canvas2 = createCanvas(width, height); + const ctx2 = canvas2.getContext('2d'); + // https://developer.mozilla.org/en-US/docs/Web/API/ImageData/ImageData + // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/putImageData + ctx2.putImageData(new nodeCanvas.ImageData(Uint8ClampedArray.from(frameRaw), width, height), 0, 0); + // require('fs').writeFileSync(`${Math.floor(Math.random() * 1e12)}.png`, canvas2.toBuffer('image/png')); + + ctx.drawImage(canvas2, 0, 0); + }); + + return canvasToRgba(ctx); +} + +async function createFabricFrameSource(func, { width, height, ...rest }) { + const onInit = async ({ canvas }) => func(({ width, height, fabric, canvas, ...rest })); + + let canvas = new fabric.StaticCanvas(null, { width, height }); + + const { onRender = () => {}, onClose = () => {} } = await onInit({ canvas }) || {}; + + async function readNextFrame(progress) { + await onRender(progress); + + canvas.renderAll(); + + const rgba = fabricCanvasToRgba(canvas); + + canvas.clear(); + // canvas.dispose(); + return rgba; + } + + return { + readNextFrame, + close: () => { + // https://stackoverflow.com/questions/19030174/how-to-manage-memory-in-case-of-multiple-fabric-js-canvas + canvas.dispose(); + canvas = undefined; + onClose(); + }, + }; +} + +async function imageFrameSource({ verbose, params, width, height, canvas }) { + if (verbose) console.log('Loading', params.path); + + const imgData = await new Promise((resolve) => fabric.util.loadImage(fileUrl(params.path), resolve)); + + const getImg = () => new fabric.Image(imgData, { + originX: 'center', + originY: 'center', + left: width / 2, + top: height / 2, + }); + + // Blurred version + const blurredImg = getImg(); + blurredImg.filters = [new fabric.Image.filters.Resize({ scaleX: 0.01, scaleY: 0.01 })]; + blurredImg.applyFilters(); + + if (blurredImg.height > blurredImg.width) blurredImg.scaleToWidth(width); + else blurredImg.scaleToHeight(height); + + + async function onRender(progress) { + const { zoomDirection = 'in', zoomAmount = 0.1 } = params; + + const img = getImg(); + + const scaleFactor = zoomDirection === 'in' ? (1 + progress * zoomAmount) : (1 + zoomAmount * (1 - progress)); + if (img.height > img.width) img.scaleToHeight(height * scaleFactor); + else img.scaleToWidth(width * scaleFactor); + + canvas.add(blurredImg); + canvas.add(img); + } + + function onClose() { + blurredImg.dispose(); + // imgData.dispose(); + } + + return { onRender, onClose }; +} + +async function fillColorFrameSource({ canvas, params }) { + const { color } = params; + + const randomColor = getRandomColors(1)[0]; + + async function onRender() { + // eslint-disable-next-line no-param-reassign + canvas.backgroundColor = color || randomColor; + } + + return { onRender }; +} + +function getRekt(width, height) { + // width and height with room to rotate + return new fabric.Rect({ originX: 'center', originY: 'center', left: width / 2, top: height / 2, width: width * 2, height: height * 2 }); +} + +async function radialGradientFrameSource({ canvas, width, height, params }) { + const { colors: inColors } = params; + + const randomColors = getRandomGradient(); + + async function onRender(progress) { + // console.log('progress', progress); + + const max = Math.max(width, height); + + const colors = inColors && inColors.length === 2 ? inColors : randomColors; + + const r1 = 0; + const r2 = max * (1 + progress) * 0.6; + + const rect = getRekt(width, height); + + const cx = 0.5 * rect.width; + const cy = 0.5 * rect.height; + + rect.setGradient('fill', { + type: 'radial', + r1, + r2, + x1: cx, + y1: cy, + x2: cx, + y2: cy, + colorStops: { + 0: colors[0], + 1: colors[1], + }, + }); + + canvas.add(rect); + } + + return { onRender }; +} + +async function linearGradientFrameSource({ canvas, width, height, params }) { + const { colors: inColors } = params; + + const randomColors = getRandomGradient(); + const colors = inColors && inColors.length === 2 ? inColors : randomColors; + + async function onRender(progress) { + const rect = getRekt(width, height); + + rect.setGradient('fill', { + x1: 0, + y1: 0, + x2: width, + y2: height, + colorStops: { + 0: colors[0], + 1: colors[1], + }, + }); + + rect.rotate(progress * 30); + canvas.add(rect); + } + + return { onRender }; +} + +async function subtitleFrameSource({ canvas, width, height, params }) { + const { text, textColor = '#ffffff', fontFamily = 'sans-serif' } = params; + + async function onRender(progress) { + const easedProgress = easeOutExpo(Math.min(progress, 1)); + + const min = Math.min(width, height); + const padding = 0.05 * min; + + const textBox = new fabric.Textbox(text, { + fill: textColor, + fontFamily, + + fontSize: min / 20, + textAlign: 'left', + width: width - padding * 2, + originX: 'center', + originY: 'bottom', + left: (width / 2) + (-1 + easedProgress) * padding, + top: height - padding, + opacity: easedProgress, + }); + + const rect = new fabric.Rect({ + left: 0, + width, + height: textBox.height + padding * 2, + top: height, + originY: 'bottom', + fill: 'rgba(0,0,0,0.2)', + opacity: easedProgress, + }); + + canvas.add(rect); + canvas.add(textBox); + } + + return { onRender }; +} + +async function titleFrameSource({ canvas, width, height, params }) { + const { text, textColor = '#ffffff', fontFamily = 'sans-serif', position = 'center' } = params; + + async function onRender(progress) { + // console.log('progress', progress); + + const min = Math.min(width, height); + + const fontSize = Math.round(min * 0.1); + + const scale = (1 + progress * 0.2).toFixed(4); + + const textBox = new fabric.Textbox(text, { + fill: textColor, + fontFamily, + fontSize, + textAlign: 'center', + width: width * 0.8, + }); + + const textImage = await new Promise((r) => textBox.cloneAsImage(r)); + + let originY = 'center'; + let top = height / 2; + if (position === 'top') { + originY = 'top'; + top = height * 0.05; + } else if (position === 'bottom') { + originY = 'bottom'; + top = height; + } + + textImage.set({ + originX: 'center', + originY, + left: width / 2, + top, + scaleX: scale, + scaleY: scale, + }); + canvas.add(textImage); + } + + return { onRender }; +} + +async function createCustomCanvasFrameSource({ width, height, params }) { + const canvas = createCanvas(width, height); + const context = canvas.getContext('2d'); + + const { onClose, onRender } = await params.func(({ width, height, canvas })); + + async function readNextFrame(progress) { + context.clearRect(0, 0, canvas.width, canvas.height); + await onRender(progress); + // require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png')); + return canvasToRgba(context); + } + + return { + readNextFrame, + // Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668 + close: onClose, + }; +} + +async function customFabricFrameSource({ canvas, width, height, params }) { + return params.func(({ width, height, fabric, canvas })); +} + +function registerFont(...args) { + fabric.nodeCanvas.registerFont(...args); +} + +module.exports = { + mergeFrames, + registerFont, + createFabricFrameSource, + createCustomCanvasFrameSource, + + customFabricFrameSource, + subtitleFrameSource, + titleFrameSource, + fillColorFrameSource, + radialGradientFrameSource, + linearGradientFrameSource, + imageFrameSource, +}; diff --git a/sources/frameSource.js b/sources/frameSource.js new file mode 100644 index 0000000..dd4b23e --- /dev/null +++ b/sources/frameSource.js @@ -0,0 +1,61 @@ +const assert = require('assert'); +const pMap = require('p-map'); + +const { mergeFrames, customFabricFrameSource, createCustomCanvasFrameSource, titleFrameSource, subtitleFrameSource, imageFrameSource, linearGradientFrameSource, radialGradientFrameSource, fillColorFrameSource, createFabricFrameSource } = require('./fabricFrameSource'); +const createVideoFrameSource = require('./videoFrameSource'); +const { createGlFrameSource } = require('./glFrameSource'); + + +async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, enableFfmpegLog, framerateStr }) { + const { layers, duration } = clip; + + const frameSources = await pMap(layers, async (layer, layerIndex) => { + const { type, ...params } = layer; + console.log('createFrameSource', type, 'clip', clipIndex, 'layer', layerIndex); + + const frameSourceFuncs = { + video: createVideoFrameSource, + gl: createGlFrameSource, + canvas: createCustomCanvasFrameSource, + fabric: async (opts) => createFabricFrameSource(customFabricFrameSource, opts), + image: async (opts) => createFabricFrameSource(imageFrameSource, opts), + title: async (opts) => createFabricFrameSource(titleFrameSource, opts), + subtitle: async (opts) => createFabricFrameSource(subtitleFrameSource, opts), + 'linear-gradient': async (opts) => createFabricFrameSource(linearGradientFrameSource, opts), + 'radial-gradient': async (opts) => createFabricFrameSource(radialGradientFrameSource, opts), + 'fill-color': async (opts) => createFabricFrameSource(fillColorFrameSource, opts), + }; + assert(frameSourceFuncs[type], `Invalid type ${type}`); + + const createFrameSourceFunc = frameSourceFuncs[type]; + + return createFrameSourceFunc({ width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params }); + }, { concurrency: 1 }); + + async function readNextFrame(...args) { + const framesRaw = await pMap(frameSources, async (frameSource) => frameSource.readNextFrame(...args)); + // if (verbose) console.time('Merge frames'); + + const framesRawFiltered = framesRaw.filter((frameRaw) => { + if (frameRaw) return true; + console.warn('Frame source returned empty result'); + return false; + }); + const merged = mergeFrames({ width, height, framesRaw: framesRawFiltered }); + // if (verbose) console.timeEnd('Merge frames'); + return merged; + } + + async function close() { + await pMap(frameSources, async (frameSource) => frameSource.close()); + } + + return { + readNextFrame, + close, + }; +} + +module.exports = { + createFrameSource, +}; diff --git a/sources/glFrameSource.js b/sources/glFrameSource.js new file mode 100644 index 0000000..07d9a44 --- /dev/null +++ b/sources/glFrameSource.js @@ -0,0 +1,65 @@ +const GL = require('gl'); +const createShader = require('gl-shader'); +const fs = require('fs-extra'); + +// I have no idea what I'm doing but it works ¯\_(ツ)_/¯ + +async function createGlFrameSource({ width, height, channels, params }) { + const gl = GL(width, height); + + const defaultVertexSrc = ` + attribute vec2 position; + void main(void) { + gl_Position = vec4(position, 0.0, 1.0 ); + } + `; + const { vertexPath, fragmentPath, vertexSrc: vertexSrcIn, fragmentSrc: fragmentSrcIn, speed = 1 } = params; + + let fragmentSrc = fragmentSrcIn; + let vertexSrc = vertexSrcIn; + + if (fragmentPath) fragmentSrc = await fs.readFile(fragmentPath); + if (vertexPath) vertexSrc = await fs.readFile(vertexPath); + + if (!vertexSrc) vertexSrc = defaultVertexSrc; + + const shader = createShader(gl, vertexSrc, fragmentSrc); + const buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + // https://blog.mayflower.de/4584-Playing-around-with-pixel-shaders-in-WebGL.html + + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, 1, 1, -1, 1]), gl.STATIC_DRAW); + + async function readNextFrame(progress) { + shader.bind(); + + shader.attributes.position.pointer(); + + shader.uniforms.resolution = [width, height]; + shader.uniforms.time = progress * speed; + + gl.drawArrays(gl.TRIANGLE_FAN, 0, 4); + + const upsideDownArray = Buffer.allocUnsafe(width * height * channels); + gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, upsideDownArray); + const outArray = Buffer.allocUnsafe(width * height * channels); + + // Comes out upside down, flip it + for (let i = 0; i < outArray.length; i += 4) { + outArray[i + 0] = upsideDownArray[outArray.length - i + 0]; + outArray[i + 1] = upsideDownArray[outArray.length - i + 1]; + outArray[i + 2] = upsideDownArray[outArray.length - i + 2]; + outArray[i + 3] = upsideDownArray[outArray.length - i + 3]; + } + return outArray; + } + + return { + readNextFrame, + close: () => {}, + }; +} + +module.exports = { + createGlFrameSource, +}; diff --git a/sources/shared.js b/sources/shared.js new file mode 100644 index 0000000..136a42c --- /dev/null +++ b/sources/shared.js @@ -0,0 +1,21 @@ +function canvasToRgba(ctx) { + // const bgra = canvas.toBuffer('raw'); + + /* const rgba = Buffer.allocUnsafe(bgra.length); + for (let i = 0; i < bgra.length; i += 4) { + rgba[i + 0] = bgra[i + 2]; + rgba[i + 1] = bgra[i + 1]; + rgba[i + 2] = bgra[i + 0]; + rgba[i + 3] = bgra[i + 3]; + } */ + + // We cannot use toBuffer('raw') because it returns pre-multiplied alpha data (a different format) + // https://gamedev.stackexchange.com/questions/138813/whats-the-difference-between-alpha-and-premulalpha + // https://github.com/Automattic/node-canvas#image-pixel-formats-experimental + const imageData = ctx.getImageData(0, 0, ctx.canvas.width, ctx.canvas.height); + return Buffer.from(imageData.data); +} + +module.exports = { + canvasToRgba, +}; diff --git a/sources/videoFrameSource.js b/sources/videoFrameSource.js new file mode 100644 index 0000000..8696a2e --- /dev/null +++ b/sources/videoFrameSource.js @@ -0,0 +1,124 @@ +const execa = require('execa'); +const assert = require('assert'); + +module.exports = async ({ width, height, channels, framerateStr, verbose, enableFfmpegLog, params }) => { + const targetSize = width * height * channels; + + // TODO assert that we have read the correct amount of frames + + const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor } = params; + + const buf = Buffer.allocUnsafe(targetSize); + let length = 0; + // let inFrameCount = 0; + + let ptsFilter = ''; + if (framePtsFactor !== 1) { + if (verbose) console.log('framePtsFactor', framePtsFactor); + ptsFilter = `setpts=${framePtsFactor}*PTS,`; + } + + let scaleFilter; + if (resizeMode === 'stretch') scaleFilter = `scale=${width}:${height}`; + // https://superuser.com/questions/891145/ffmpeg-upscale-and-letterbox-a-video/891478 + else if (resizeMode === 'contain') scaleFilter = `scale=(iw*sar)*min(${width}/(iw*sar)\\,${height}/ih):ih*min(${width}/(iw*sar)\\,${height}/ih), pad=${width}:${height}:(${width}-iw*min(${width}/iw\\,${height}/ih))/2:(${height}-ih*min(${width}/iw\\,${height}/ih))/2:${backgroundColor}`; + // Cover: https://unix.stackexchange.com/a/192123 + else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`; + + // http://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/ + // Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4 + // https://trac.ffmpeg.org/wiki/ChangingFrameRate + const args = [ + ...(enableFfmpegLog ? [] : ['-hide_banner', '-loglevel', 'panic']), + ...(cutFrom ? ['-ss', cutFrom] : []), + '-i', path, + ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []), + '-vf', `${ptsFilter}fps=${framerateStr},${scaleFilter}`, + '-map', 'v:0', + '-vcodec', 'rawvideo', + '-pix_fmt', 'rgba', + '-f', 'image2pipe', + '-', + ]; + if (verbose) console.log(args.join(' ')); + + const ps = execa('ffmpeg', args, { encoding: null, buffer: false, stdin: 'ignore', stdout: 'pipe', stderr: process.stderr }); + + const stream = ps.stdout; + + let timeout; + let ended = false; + + const readNextFrame = () => new Promise((resolve, reject) => { + if (ended) { + resolve(); + return; + } + // console.log('Reading new frame', path); + + function onEnd() { + if (verbose) console.log(path, 'ffmpeg stream ended'); + ended = true; + resolve(); + } + + function handleChunk(chunk) { + // console.log('chunk', chunk.length); + const nCopied = length + chunk.length > targetSize ? targetSize - length : chunk.length; + chunk.copy(buf, length, 0, nCopied); + length += nCopied; + + if (length > targetSize) console.error('OOPS! Overflow', length); + + if (length >= targetSize) { + // console.log('Finished reading frame', inFrameCount, path); + const out = Buffer.from(buf); + + const restLength = chunk.length - nCopied; + if (restLength > 0) { + if (verbose) console.log('Left over data', nCopied, chunk.length, restLength); + chunk.slice(nCopied).copy(buf, 0); + length = restLength; + } else { + length = 0; + } + + // inFrameCount += 1; + + clearTimeout(timeout); + stream.pause(); + stream.removeListener('data', handleChunk); + stream.removeListener('end', onEnd); + stream.removeListener('error', reject); + resolve(out); + } + } + + timeout = setTimeout(() => { + console.warn('Timeout on read video frame'); + stream.pause(); + stream.removeListener('data', handleChunk); + stream.removeListener('end', onEnd); + stream.removeListener('error', reject); + resolve(); + }, 10000); + + stream.on('data', handleChunk); + stream.on('end', onEnd); + stream.on('error', reject); + stream.resume(); + }).then((data) => { + if (data) assert(data.length === targetSize); + return data; + }); + + const close = () => { + console.log('Close', path); + ps.cancel(); + }; + + return { + readNextFrame, + close, + }; +}; diff --git a/transitions.js b/transitions.js new file mode 100644 index 0000000..26cdcf3 --- /dev/null +++ b/transitions.js @@ -0,0 +1,71 @@ +const assert = require('assert'); + +const randomTransitionsSet = ['fade', 'fadegrayscale', 'directionalwarp', 'crosswarp', 'dreamyzoom', 'burn', 'crosszoom', 'simplezoom', 'linearblur', 'directional-left', 'directional-right', 'directional-up', 'directional-down']; + +function getRandomTransition() { + return randomTransitionsSet[Math.floor(Math.random() * randomTransitionsSet.length)]; +} + + +// https://easings.net/ + +function easeOutExpo(x) { + return x === 1 ? 1 : 1 - (2 ** (-10 * x)); +} + +function easeInOutCubic(x) { + return x < 0.5 ? 4 * x * x * x : 1 - ((-2 * x + 2) ** 3) / 2; +} + + +function getTransitionEasingFunction(easing, transitionName) { + if (easing !== null) { + if (easing) return { easeOutExpo }[easing]; + if (transitionName === 'directional') return easeOutExpo; + } + return (progress) => progress; +} + +function calcTransition(defaults, transition) { + if (transition === null) return { duration: 0 }; + + let transitionOrDefault = { + name: (transition && transition.name) || (defaults.transition && defaults.transition.name), + duration: (transition && transition.duration != null) ? transition.duration : (defaults.transition && defaults.transition.duration), + params: (transition && transition.params) || (defaults.transition && defaults.transition.params), + easing: (transition && transition.easing !== undefined) ? transition.easing : (defaults.transition && defaults.transition.easing), + }; + + assert(!transitionOrDefault.duration || transitionOrDefault.name, 'Please specify transition name or set duration to 0'); + + if (transitionOrDefault.name === 'random') { + transitionOrDefault = { easing: transitionOrDefault.easing, name: getRandomTransition(), duration: 0.5 }; + } + + const getTransitionByAlias = () => { + const aliasedTransition = { + 'directional-left': { name: 'directional', params: { direction: [1, 0] } }, + 'directional-right': { name: 'directional', params: { direction: [-1, 0] } }, + 'directional-down': { name: 'directional', params: { direction: [0, 1] } }, + 'directional-up': { name: 'directional', params: { direction: [0, -1] } }, + }[transitionOrDefault.name]; + if (aliasedTransition) return { ...transitionOrDefault, ...aliasedTransition }; + return transitionOrDefault; + }; + + const outTransition = getTransitionByAlias(); + + return { + name: outTransition.name, + duration: outTransition.duration, + params: outTransition.params, + easingFunction: getTransitionEasingFunction(outTransition.easing, outTransition.name), + }; +} + + +module.exports = { + calcTransition, + easeInOutCubic, + easeOutExpo, +}; diff --git a/util.js b/util.js new file mode 100644 index 0000000..1f5edb3 --- /dev/null +++ b/util.js @@ -0,0 +1,34 @@ +const execa = require('execa'); + +function parseFps(fps) { + const match = typeof fps === 'string' && fps.match(/^([0-9]+)\/([0-9]+)$/); + if (match) { + const num = parseInt(match[1], 10); + const den = parseInt(match[2], 10); + if (den > 0) return num / den; + } + return undefined; +} + +async function readFileInfo(p) { + const { stdout } = await execa('ffprobe', [ + '-select_streams', 'v:0', '-show_entries', 'stream', '-of', 'json', p, + ]); + const json = JSON.parse(stdout); + const stream = json.streams[0]; + return { + // numFrames: parseInt(stream.nb_frames, 10), + duration: parseFloat(stream.duration, 10), + width: stream.width, // TODO coded_width? + height: stream.height, + framerateStr: stream.r_frame_rate, + }; +} + +const multipleOf2 = (x) => (x + (x % 2)); + +module.exports = { + parseFps, + readFileInfo, + multipleOf2, +};