implement stateless video mode

index.js

@@ -483,7 +483,7 @@ module.exports = async (config = {}) => {
         continue;
       }
 
-      const newFrameSource1Data = await frameSource1.readNextFrame({ time: fromClipTime });
+      const newFrameSource1Data = await frameSource1.renderFrame({ time: fromClipTime });
       // If we got no data, use the old data
       // TODO maybe abort?
       if (newFrameSource1Data) frameSource1Data = newFrameSource1Data;
@@ -493,7 +493,7 @@ module.exports = async (config = {}) => {
 
       let outFrameData;
       if (isInTransition) {
-        const frameSource2Data = await frameSource2.readNextFrame({ time: toClipTime });
+        const frameSource2Data = await frameSource2.renderFrame({ time: toClipTime });
 
         if (frameSource2Data) {
           const progress = transitionFrameAt / transitionNumFramesSafe;

sources/fabric.js

@@ -46,7 +46,7 @@ async function createFabricFrameSource(func, { width, height, ...rest }) {
   const { onRender = () => {}, onClose = () => {} } = await onInit() || {};
 
   return {
-    readNextFrame: onRender,
+    renderFrame: onRender,
     close: onClose,
   };
 }
@@ -57,7 +57,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) {
 
   const { onClose, onRender } = await params.func(({ width, height, canvas }));
 
-  async function readNextFrame(progress) {
+  async function renderFrame(progress) {
     context.clearRect(0, 0, canvas.width, canvas.height);
     await onRender(progress);
     // require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png'));
@@ -66,7 +66,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) {
   }
 
   return {
-    readNextFrame,
+    renderFrame,
     // Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668
     close: onClose,
   };

sources/frameSource.js

@@ -43,11 +43,11 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
 
     assert(createFrameSourceFunc, `Invalid type ${type}`);
 
-    const frameSource = await createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
+    const frameSource = await createFrameSourceFunc({ statelessMode: true, ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
     return { layer, frameSource };
   }, { concurrency: 1 });
 
-  async function readNextFrame({ time }) {
+  async function renderFrame({ time }) {
     const canvas = createFabricCanvas({ width, height });
 
     // eslint-disable-next-line no-restricted-syntax
@@ -58,7 +58,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
       const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1;
 
       if (shouldDrawLayer) {
-        const rgba = await frameSource.readNextFrame(offsetProgress, canvas);
+        const rgba = await frameSource.renderFrame(offsetProgress, canvas);
         // Frame sources can either render to the provided canvas and return nothing
         // OR return an raw RGBA blob which will be drawn onto the canvas
         if (rgba) {
@@ -82,7 +82,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
   }
 
   return {
-    readNextFrame,
+    renderFrame,
     close,
   };
 }

sources/glFrameSource.js

@@ -30,7 +30,7 @@ async function createGlFrameSource({ width, height, channels, params }) {
 
   gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, 1, 1, -1, 1]), gl.STATIC_DRAW);
 
-  async function readNextFrame(progress) {
+  async function renderFrame(progress) {
     shader.bind();
 
     shader.attributes.position.pointer();
@@ -55,7 +55,7 @@ async function createGlFrameSource({ width, height, channels, params }) {
   }
 
   return {
-    readNextFrame,
+    renderFrame,
     close: () => {},
   };
 }

sources/videoFrameSource.js

@@ -4,130 +4,165 @@ const assert = require('assert');
 const { getFfmpegCommonArgs } = require('../ffmpeg');
 const { readFileStreams } = require('../util');
 
-module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
+module.exports = async ({ statelessMode, width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
   const targetSize = width * height * channels;
 
-  // TODO assert that we have read the correct amount of frames
+  const buf = Buffer.allocUnsafe(targetSize);
 
-  const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor } = params;
+  // TODO assert that we have read the correct amount of frames
 
-  const buf = Buffer.allocUnsafe(targetSize);
-  let length = 0;
-  // let inFrameCount = 0;
+  const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor, inputDuration } = params;
 
-  let ptsFilter = '';
-  if (framePtsFactor !== 1) {
-    if (verbose) console.log('framePtsFactor', framePtsFactor);
-    ptsFilter = `setpts=${framePtsFactor}*PTS,`;
-  }
+  let initializing;
 
-  let scaleFilter;
-  if (resizeMode === 'stretch') scaleFilter = `scale=${width}:${height}`;
-  // https://superuser.com/questions/891145/ffmpeg-upscale-and-letterbox-a-video/891478
-  else if (resizeMode === 'contain' || resizeMode === 'contain-blur') scaleFilter = `scale=(iw*sar)*min(${width}/(iw*sar)\\,${height}/ih):ih*min(${width}/(iw*sar)\\,${height}/ih), pad=${width}:${height}:(${width}-iw*min(${width}/iw\\,${height}/ih))/2:(${height}-ih*min(${width}/iw\\,${height}/ih))/2:${backgroundColor}`;
-  // Cover: https://unix.stackexchange.com/a/192123
-  else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`;
-
-  // https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/
-  const streams = await readFileStreams(ffprobePath, path);
-  const firstVideoStream = streams.find((s) => s.codec_type === 'video');
-  // https://superuser.com/a/1116905/658247
-  const inputCodecArgs = ['vp8', 'vp9'].includes(firstVideoStream.codec_name) ? ['-vcodec', 'libvpx'] : [];
-
-  // http://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/
-  // Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4
-  // https://trac.ffmpeg.org/wiki/ChangingFrameRate
-  const args = [
-    ...getFfmpegCommonArgs({ enableFfmpegLog }),
-    ...inputCodecArgs,
-    ...(cutFrom ? ['-ss', cutFrom] : []),
-    '-i', path,
-    ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []),
-    '-vf', `${ptsFilter}fps=${framerateStr},${scaleFilter}`,
-    '-map', 'v:0',
-    '-vcodec', 'rawvideo',
-    '-pix_fmt', 'rgba',
-    '-f', 'image2pipe',
-    '-',
-  ];
-  if (verbose) console.log(args.join(' '));
-
-  const ps = execa(ffmpegPath, args, { encoding: null, buffer: false, stdin: 'ignore', stdout: 'pipe', stderr: process.stderr });
-
-  const stream = ps.stdout;
+  let stream;
+  let ps;
+  let length;
 
   let timeout;
-  let ended = false;
+  let ended;
+
+  async function restartProcess(cutFromWithProgress) {
+    console.log('(Re)start process', { cutFromWithProgress });
 
-  stream.once('end', () => {
+    if (initializing) throw new Error('Already initializing video');
+    initializing = true;
+
+    stream = undefined;
+    if (ps) ps.cancel();
+    ps = undefined;
+    length = 0;
     clearTimeout(timeout);
-    if (verbose) console.log(path, 'ffmpeg video stream ended');
-    ended = true;
-  });
-
-  const readNextFrame = () => new Promise((resolve, reject) => {
-    if (ended) {
-      console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
-      resolve();
-      return;
-    }
-    // console.log('Reading new frame', path);
+    timeout = undefined;
+    ended = false;
 
-    function onEnd() {
-      resolve();
+    // let inFrameCount = 0;
+
+    let ptsFilter = '';
+    if (framePtsFactor !== 1) {
+      if (verbose) console.log('framePtsFactor', framePtsFactor);
+      ptsFilter = `setpts=${framePtsFactor}*PTS,`;
     }
 
-    function cleanup() {
-      stream.pause();
-      // eslint-disable-next-line no-use-before-define
-      stream.removeListener('data', handleChunk);
-      stream.removeListener('end', onEnd);
-      stream.removeListener('error', reject);
+    let scaleFilter;
+    if (resizeMode === 'stretch') scaleFilter = `scale=${width}:${height}`;
+    // https://superuser.com/questions/891145/ffmpeg-upscale-and-letterbox-a-video/891478
+    else if (resizeMode === 'contain' || resizeMode === 'contain-blur') scaleFilter = `scale=(iw*sar)*min(${width}/(iw*sar)\\,${height}/ih):ih*min(${width}/(iw*sar)\\,${height}/ih), pad=${width}:${height}:(${width}-iw*min(${width}/iw\\,${height}/ih))/2:(${height}-ih*min(${width}/iw\\,${height}/ih))/2:${backgroundColor}`;
+    // Cover: https://unix.stackexchange.com/a/192123
+    else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`;
+
+    // https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/
+    const streams = await readFileStreams(ffprobePath, path);
+    const firstVideoStream = streams.find((s) => s.codec_type === 'video');
+    // https://superuser.com/a/1116905/658247
+    const inputCodecArgs = ['vp8', 'vp9'].includes(firstVideoStream.codec_name) ? ['-vcodec', 'libvpx'] : [];
+
+    // http://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/
+    // Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4
+    // https://trac.ffmpeg.org/wiki/ChangingFrameRate
+    const args = [
+      ...getFfmpegCommonArgs({ enableFfmpegLog }),
+      ...inputCodecArgs,
+      ...(cutFromWithProgress ? ['-ss', cutFromWithProgress] : []),
+      '-i', path,
+      // TODO -t maybe not needed as we will close the stream when we don't need more?
+      // ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []),
+      ...(statelessMode ? ['-vframes', '1'] : []),
+      '-vf', `${ptsFilter}fps=${framerateStr},${scaleFilter}`,
+      '-map', 'v:0',
+      '-vcodec', 'rawvideo',
+      '-pix_fmt', 'rgba',
+      '-f', 'image2pipe',
+      '-',
+    ];
+    if (verbose) console.log(args.join(' '));
+
+    ps = execa(ffmpegPath, args, { encoding: null, buffer: false, stdin: 'ignore', stdout: 'pipe', stderr: process.stderr });
+
+    stream = ps.stdout;
+
+    stream.once('end', () => {
+      clearTimeout(timeout);
+      if (verbose) console.log(path, 'ffmpeg video stream ended');
+      ended = true;
+    });
+
+    initializing = false; // TODO try finally
+  }
+
+  // await restartProcess();
+
+  const renderFrame = async (progress) => {
+    // console.log(progress)
+    if (statelessMode || progress === 0) {
+      const cutFromWithProgress = (cutFrom || 0) + (progress * inputDuration);
+      await restartProcess(cutFromWithProgress);
     }
 
-    function handleChunk(chunk) {
-      // console.log('chunk', chunk.length);
-      const nCopied = length + chunk.length > targetSize ? targetSize - length : chunk.length;
-      chunk.copy(buf, length, 0, nCopied);
-      length += nCopied;
-
-      if (length > targetSize) console.error('Video data overflow', length);
-
-      if (length >= targetSize) {
-        // console.log('Finished reading frame', inFrameCount, path);
-        const out = Buffer.from(buf);
-
-        const restLength = chunk.length - nCopied;
-        if (restLength > 0) {
-          // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
-          chunk.slice(nCopied).copy(buf, 0);
-          length = restLength;
-        } else {
-          length = 0;
-        }
+    const data = await new Promise((resolve, reject) => {
+      if (ended) {
+        console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
+        resolve();
+        return;
+      }
+      // console.log('Reading new frame', path);
 
-        // inFrameCount += 1;
+      function onEnd() {
+        resolve();
+      }
 
-        clearTimeout(timeout);
-        cleanup();
-        resolve(out);
+      function cleanup() {
+        stream.pause();
+        // eslint-disable-next-line no-use-before-define
+        stream.removeListener('data', handleChunk);
+        stream.removeListener('end', onEnd);
+        stream.removeListener('error', reject);
       }
-    }
 
-    timeout = setTimeout(() => {
-      console.warn('Timeout on read video frame');
-      cleanup();
-      resolve();
-    }, 60000);
-
-    stream.on('data', handleChunk);
-    stream.on('end', onEnd);
-    stream.on('error', reject);
-    stream.resume();
-  }).then((data) => {
-    if (data) assert(data.length === targetSize);
+      function handleChunk(chunk) {
+        // console.log('chunk', chunk.length);
+        const nCopied = length + chunk.length > targetSize ? targetSize - length : chunk.length;
+        chunk.copy(buf, length, 0, nCopied);
+        length += nCopied;
+
+        if (length > targetSize) console.error('Video data overflow', length);
+
+        if (length >= targetSize) {
+          // console.log('Finished reading frame', inFrameCount, path);
+          const out = Buffer.from(buf);
+
+          const restLength = chunk.length - nCopied;
+          if (restLength > 0) {
+            // if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
+            chunk.slice(nCopied).copy(buf, 0);
+            length = restLength;
+          } else {
+            length = 0;
+          }
+
+          // inFrameCount += 1;
+
+          clearTimeout(timeout);
+          cleanup();
+          resolve(out);
+        }
+      }
+
+      timeout = setTimeout(() => {
+        console.warn('Timeout on read video frame');
+        cleanup();
+        resolve();
+      }, 60000);
+
+      stream.on('data', handleChunk);
+      stream.on('end', onEnd);
+      stream.on('error', reject);
+      stream.resume();
+    });
+
+    if (data) assert.equal(data.length, targetSize);
     return data;
-  });
+  };
 
   const close = () => {
     if (verbose) console.log('Close', path);
@@ -135,7 +170,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
   };
 
   return {
-    readNextFrame,
+    renderFrame,
     close,
   };
 };