implement stateless video mode

index.js

@@ -483,7 +483,7 @@ module.exports = async (config = {}) => {
         continue;
       }
 
-      const newFrameSource1Data = await frameSource1.readNextFrame({ time: fromClipTime });
+      const newFrameSource1Data = await frameSource1.renderFrame({ time: fromClipTime });
       // If we got no data, use the old data
       // TODO maybe abort?
       if (newFrameSource1Data) frameSource1Data = newFrameSource1Data;
@@ -493,7 +493,7 @@ module.exports = async (config = {}) => {
 
       let outFrameData;
       if (isInTransition) {
-        const frameSource2Data = await frameSource2.readNextFrame({ time: toClipTime });
+        const frameSource2Data = await frameSource2.renderFrame({ time: toClipTime });
 
         if (frameSource2Data) {
           const progress = transitionFrameAt / transitionNumFramesSafe;

sources/fabric.js

@@ -46,7 +46,7 @@ async function createFabricFrameSource(func, { width, height, ...rest }) {
   const { onRender = () => {}, onClose = () => {} } = await onInit() || {};
 
   return {
-    readNextFrame: onRender,
+    renderFrame: onRender,
     close: onClose,
   };
 }
@@ -57,7 +57,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) {
 
   const { onClose, onRender } = await params.func(({ width, height, canvas }));
 
-  async function readNextFrame(progress) {
+  async function renderFrame(progress) {
     context.clearRect(0, 0, canvas.width, canvas.height);
     await onRender(progress);
     // require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png'));
@@ -66,7 +66,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) {
   }
 
   return {
-    readNextFrame,
+    renderFrame,
     // Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668
     close: onClose,
   };

sources/frameSource.js

@@ -43,11 +43,11 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
 
     assert(createFrameSourceFunc, `Invalid type ${type}`);
 
-    const frameSource = await createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
+    const frameSource = await createFrameSourceFunc({ statelessMode: true, ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
     return { layer, frameSource };
   }, { concurrency: 1 });
 
-  async function readNextFrame({ time }) {
+  async function renderFrame({ time }) {
     const canvas = createFabricCanvas({ width, height });
 
     // eslint-disable-next-line no-restricted-syntax
@@ -58,7 +58,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
       const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1;
 
       if (shouldDrawLayer) {
-        const rgba = await frameSource.readNextFrame(offsetProgress, canvas);
+        const rgba = await frameSource.renderFrame(offsetProgress, canvas);
         // Frame sources can either render to the provided canvas and return nothing
         // OR return an raw RGBA blob which will be drawn onto the canvas
         if (rgba) {
@@ -82,7 +82,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
   }
 
   return {
-    readNextFrame,
+    renderFrame,
     close,
   };
 }

sources/glFrameSource.js

@@ -30,7 +30,7 @@ async function createGlFrameSource({ width, height, channels, params }) {
 
   gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, 1, 1, -1, 1]), gl.STATIC_DRAW);
 
-  async function readNextFrame(progress) {
+  async function renderFrame(progress) {
     shader.bind();
 
     shader.attributes.position.pointer();
@@ -55,7 +55,7 @@ async function createGlFrameSource({ width, height, channels, params }) {
   }
 
   return {
-    readNextFrame,
+    renderFrame,
     close: () => {},
   };
 }

sources/videoFrameSource.js

@@ -4,15 +4,38 @@ const assert = require('assert');
 const { getFfmpegCommonArgs } = require('../ffmpeg');
 const { readFileStreams } = require('../util');
 
-module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
+module.exports = async ({ statelessMode, width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
   const targetSize = width * height * channels;
 
+  const buf = Buffer.allocUnsafe(targetSize);
+
   // TODO assert that we have read the correct amount of frames
 
-  const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor } = params;
+  const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor, inputDuration } = params;
+
+  let initializing;
+
+  let stream;
+  let ps;
+  let length;
+
+  let timeout;
+  let ended;
+
+  async function restartProcess(cutFromWithProgress) {
+    console.log('(Re)start process', { cutFromWithProgress });
+
+    if (initializing) throw new Error('Already initializing video');
+    initializing = true;
+
+    stream = undefined;
+    if (ps) ps.cancel();
+    ps = undefined;
+    length = 0;
+    clearTimeout(timeout);
+    timeout = undefined;
+    ended = false;
 
-  const buf = Buffer.allocUnsafe(targetSize);
-  let length = 0;
     // let inFrameCount = 0;
 
     let ptsFilter = '';
@@ -40,9 +63,11 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
     const args = [
       ...getFfmpegCommonArgs({ enableFfmpegLog }),
       ...inputCodecArgs,
-    ...(cutFrom ? ['-ss', cutFrom] : []),
+      ...(cutFromWithProgress ? ['-ss', cutFromWithProgress] : []),
       '-i', path,
-    ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []),
+      // TODO -t maybe not needed as we will close the stream when we don't need more?
+      // ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []),
+      ...(statelessMode ? ['-vframes', '1'] : []),
       '-vf', `${ptsFilter}fps=${framerateStr},${scaleFilter}`,
       '-map', 'v:0',
       '-vcodec', 'rawvideo',
@@ -52,12 +77,9 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
     ];
     if (verbose) console.log(args.join(' '));
 
-  const ps = execa(ffmpegPath, args, { encoding: null, buffer: false, stdin: 'ignore', stdout: 'pipe', stderr: process.stderr });
+    ps = execa(ffmpegPath, args, { encoding: null, buffer: false, stdin: 'ignore', stdout: 'pipe', stderr: process.stderr });
 
-  const stream = ps.stdout;
-
-  let timeout;
-  let ended = false;
+    stream = ps.stdout;
 
     stream.once('end', () => {
       clearTimeout(timeout);
@@ -65,7 +87,19 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
       ended = true;
     });
 
-  const readNextFrame = () => new Promise((resolve, reject) => {
+    initializing = false; // TODO try finally
+  }
+
+  // await restartProcess();
+
+  const renderFrame = async (progress) => {
+    // console.log(progress)
+    if (statelessMode || progress === 0) {
+      const cutFromWithProgress = (cutFrom || 0) + (progress * inputDuration);
+      await restartProcess(cutFromWithProgress);
+    }
+
+    const data = await new Promise((resolve, reject) => {
       if (ended) {
         console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
         resolve();
@@ -124,18 +158,19 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
       stream.on('end', onEnd);
       stream.on('error', reject);
       stream.resume();
-  }).then((data) => {
-    if (data) assert(data.length === targetSize);
-    return data;
     });
 
+    if (data) assert.equal(data.length, targetSize);
+    return data;
+  };
+
   const close = () => {
     if (verbose) console.log('Close', path);
     ps.cancel();
   };
 
   return {
-    readNextFrame,
+    renderFrame,
     close,
   };
 };