2020-2021-Pair-test-1-Tp-test
/
Michel-Beaussart-Main-git-repo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

implement stateless video mode

stateless
Mikael Finstad 6 years ago
parent
ccd42cc2da
commit
6058ee13ec
5 changed files with 151 additions and 116 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      index.js
  2. 6
      sources/fabric.js
  3. 8
      sources/frameSource.js
  4. 4
      sources/glFrameSource.js
  5. 245
      sources/videoFrameSource.js

4
index.js
View File

@ -483,7 +483,7 @@ module.exports = async (config = {}) => {
continue;
}
const newFrameSource1Data = await frameSource1.readNextFrame({ time: fromClipTime });
const newFrameSource1Data = await frameSource1.renderFrame({ time: fromClipTime });
// If we got no data, use the old data
// TODO maybe abort?
if (newFrameSource1Data) frameSource1Data = newFrameSource1Data;
@ -493,7 +493,7 @@ module.exports = async (config = {}) => {
let outFrameData;
if (isInTransition) {
const frameSource2Data = await frameSource2.readNextFrame({ time: toClipTime });
const frameSource2Data = await frameSource2.renderFrame({ time: toClipTime });
if (frameSource2Data) {
const progress = transitionFrameAt / transitionNumFramesSafe;

6
sources/fabric.js
View File

@ -46,7 +46,7 @@ async function createFabricFrameSource(func, { width, height, ...rest }) {
const { onRender = () => {}, onClose = () => {} } = await onInit() || {};
return {
readNextFrame: onRender,
renderFrame: onRender,
close: onClose,
};
}
@ -57,7 +57,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) {
const { onClose, onRender } = await params.func(({ width, height, canvas }));
async function readNextFrame(progress) {
async function renderFrame(progress) {
context.clearRect(0, 0, canvas.width, canvas.height);
await onRender(progress);
// require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png'));
@ -66,7 +66,7 @@ async function createCustomCanvasFrameSource({ width, height, params }) {
}
return {
readNextFrame,
renderFrame,
// Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668
close: onClose,
};

8
sources/frameSource.js
View File

@ -43,11 +43,11 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
assert(createFrameSourceFunc, `Invalid type ${type}`);
const frameSource = await createFrameSourceFunc({ ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
const frameSource = await createFrameSourceFunc({ statelessMode: true, ffmpegPath, ffprobePath, width, height, duration, channels, verbose, enableFfmpegLog, framerateStr, params });
return { layer, frameSource };
}, { concurrency: 1 });
async function readNextFrame({ time }) {
async function renderFrame({ time }) {
const canvas = createFabricCanvas({ width, height });
// eslint-disable-next-line no-restricted-syntax
@ -58,7 +58,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1;
if (shouldDrawLayer) {
const rgba = await frameSource.readNextFrame(offsetProgress, canvas);
const rgba = await frameSource.renderFrame(offsetProgress, canvas);
// Frame sources can either render to the provided canvas and return nothing
// OR return an raw RGBA blob which will be drawn onto the canvas
if (rgba) {
@ -82,7 +82,7 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
}
return {
readNextFrame,
renderFrame,
close,
};
}

4
sources/glFrameSource.js
View File

@ -30,7 +30,7 @@ async function createGlFrameSource({ width, height, channels, params }) {
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, 1, 1, -1, 1]), gl.STATIC_DRAW);
async function readNextFrame(progress) {
async function renderFrame(progress) {
shader.bind();
shader.attributes.position.pointer();
@ -55,7 +55,7 @@ async function createGlFrameSource({ width, height, channels, params }) {
}
return {
readNextFrame,
renderFrame,
close: () => {},
};
}

245
sources/videoFrameSource.js
View File

@ -4,130 +4,165 @@ const assert = require('assert');
const { getFfmpegCommonArgs } = require('../ffmpeg');
const { readFileStreams } = require('../util');
module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
module.exports = async ({ statelessMode, width, height, channels, framerateStr, verbose, ffmpegPath, ffprobePath, enableFfmpegLog, params }) => {
const targetSize = width * height * channels;
// TODO assert that we have read the correct amount of frames
const buf = Buffer.allocUnsafe(targetSize);
const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor } = params;
// TODO assert that we have read the correct amount of frames
const buf = Buffer.allocUnsafe(targetSize);
let length = 0;
// let inFrameCount = 0;
const { path, cutFrom, cutTo, resizeMode = 'cover', backgroundColor = '#000000', framePtsFactor, inputDuration } = params;
let ptsFilter = '';
if (framePtsFactor !== 1) {
if (verbose) console.log('framePtsFactor', framePtsFactor);
ptsFilter = `setpts=${framePtsFactor}*PTS,`;
}
let initializing;
let scaleFilter;
if (resizeMode === 'stretch') scaleFilter = `scale=${width}:${height}`;
// https://superuser.com/questions/891145/ffmpeg-upscale-and-letterbox-a-video/891478
else if (resizeMode === 'contain' || resizeMode === 'contain-blur') scaleFilter = `scale=(iw*sar)*min(${width}/(iw*sar)\\,${height}/ih):ih*min(${width}/(iw*sar)\\,${height}/ih), pad=${width}:${height}:(${width}-iw*min(${width}/iw\\,${height}/ih))/2:(${height}-ih*min(${width}/iw\\,${height}/ih))/2:${backgroundColor}`;
// Cover: https://unix.stackexchange.com/a/192123
else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`;
// https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/
const streams = await readFileStreams(ffprobePath, path);
const firstVideoStream = streams.find((s) => s.codec_type === 'video');
// https://superuser.com/a/1116905/658247
const inputCodecArgs = ['vp8', 'vp9'].includes(firstVideoStream.codec_name) ? ['-vcodec', 'libvpx'] : [];
// http://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/
// Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4
// https://trac.ffmpeg.org/wiki/ChangingFrameRate
const args = [
...getFfmpegCommonArgs({ enableFfmpegLog }),
...inputCodecArgs,
...(cutFrom ? ['-ss', cutFrom] : []),
'-i', path,
...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []),
'-vf', `${ptsFilter}fps=${framerateStr},${scaleFilter}`,
'-map', 'v:0',
'-vcodec', 'rawvideo',
'-pix_fmt', 'rgba',
'-f', 'image2pipe',
'-',
];
if (verbose) console.log(args.join(' '));
const ps = execa(ffmpegPath, args, { encoding: null, buffer: false, stdin: 'ignore', stdout: 'pipe', stderr: process.stderr });
const stream = ps.stdout;
let stream;
let ps;
let length;
let timeout;
let ended = false;
let ended;
async function restartProcess(cutFromWithProgress) {
console.log('(Re)start process', { cutFromWithProgress });
stream.once('end', () => {
if (initializing) throw new Error('Already initializing video');
initializing = true;
stream = undefined;
if (ps) ps.cancel();
ps = undefined;
length = 0;
clearTimeout(timeout);
if (verbose) console.log(path, 'ffmpeg video stream ended');
ended = true;
});
const readNextFrame = () => new Promise((resolve, reject) => {
if (ended) {
console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
resolve();
return;
}
// console.log('Reading new frame', path);
timeout = undefined;
ended = false;
function onEnd() {
resolve();
// let inFrameCount = 0;
let ptsFilter = '';
if (framePtsFactor !== 1) {
if (verbose) console.log('framePtsFactor', framePtsFactor);
ptsFilter = `setpts=${framePtsFactor}*PTS,`;
}
function cleanup() {
stream.pause();
// eslint-disable-next-line no-use-before-define
stream.removeListener('data', handleChunk);
stream.removeListener('end', onEnd);
stream.removeListener('error', reject);
let scaleFilter;
if (resizeMode === 'stretch') scaleFilter = `scale=${width}:${height}`;
// https://superuser.com/questions/891145/ffmpeg-upscale-and-letterbox-a-video/891478
else if (resizeMode === 'contain' || resizeMode === 'contain-blur') scaleFilter = `scale=(iw*sar)*min(${width}/(iw*sar)\\,${height}/ih):ih*min(${width}/(iw*sar)\\,${height}/ih), pad=${width}:${height}:(${width}-iw*min(${width}/iw\\,${height}/ih))/2:(${height}-ih*min(${width}/iw\\,${height}/ih))/2:${backgroundColor}`;
// Cover: https://unix.stackexchange.com/a/192123
else scaleFilter = `scale=(iw*sar)*max(${width}/(iw*sar)\\,${height}/ih):ih*max(${width}/(iw*sar)\\,${height}/ih),crop=${width}:${height}`;
// https://forum.unity.com/threads/settings-for-importing-a-video-with-an-alpha-channel.457657/
const streams = await readFileStreams(ffprobePath, path);
const firstVideoStream = streams.find((s) => s.codec_type === 'video');
// https://superuser.com/a/1116905/658247
const inputCodecArgs = ['vp8', 'vp9'].includes(firstVideoStream.codec_name) ? ['-vcodec', 'libvpx'] : [];
// http://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/
// Testing: ffmpeg -i 'vid.mov' -t 1 -vcodec rawvideo -pix_fmt rgba -f image2pipe - | ffmpeg -f rawvideo -vcodec rawvideo -pix_fmt rgba -s 2166x1650 -i - -vf format=yuv420p -vcodec libx264 -y out.mp4
// https://trac.ffmpeg.org/wiki/ChangingFrameRate
const args = [
...getFfmpegCommonArgs({ enableFfmpegLog }),
...inputCodecArgs,
...(cutFromWithProgress ? ['-ss', cutFromWithProgress] : []),
'-i', path,
// TODO -t maybe not needed as we will close the stream when we don't need more?
// ...(cutTo ? ['-t', (cutTo - cutFrom) * framePtsFactor] : []),
...(statelessMode ? ['-vframes', '1'] : []),
'-vf', `${ptsFilter}fps=${framerateStr},${scaleFilter}`,
'-map', 'v:0',
'-vcodec', 'rawvideo',
'-pix_fmt', 'rgba',
'-f', 'image2pipe',
'-',
];
if (verbose) console.log(args.join(' '));
ps = execa(ffmpegPath, args, { encoding: null, buffer: false, stdin: 'ignore', stdout: 'pipe', stderr: process.stderr });
stream = ps.stdout;
stream.once('end', () => {
clearTimeout(timeout);
if (verbose) console.log(path, 'ffmpeg video stream ended');
ended = true;
});
initializing = false; // TODO try finally
}
// await restartProcess();
const renderFrame = async (progress) => {
// console.log(progress)
if (statelessMode || progress === 0) {
const cutFromWithProgress = (cutFrom || 0) + (progress * inputDuration);
await restartProcess(cutFromWithProgress);
}
function handleChunk(chunk) {
// console.log('chunk', chunk.length);
const nCopied = length + chunk.length > targetSize ? targetSize - length : chunk.length;
chunk.copy(buf, length, 0, nCopied);
length += nCopied;
if (length > targetSize) console.error('Video data overflow', length);
if (length >= targetSize) {
// console.log('Finished reading frame', inFrameCount, path);
const out = Buffer.from(buf);
const restLength = chunk.length - nCopied;
if (restLength > 0) {
// if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
chunk.slice(nCopied).copy(buf, 0);
length = restLength;
} else {
length = 0;
}
const data = await new Promise((resolve, reject) => {
if (ended) {
console.log(path, 'Tried to read next video frame after ffmpeg video stream ended');
resolve();
return;
}
// console.log('Reading new frame', path);
// inFrameCount += 1;
function onEnd() {
resolve();
}
clearTimeout(timeout);
cleanup();
resolve(out);
function cleanup() {
stream.pause();
// eslint-disable-next-line no-use-before-define
stream.removeListener('data', handleChunk);
stream.removeListener('end', onEnd);
stream.removeListener('error', reject);
}
}
timeout = setTimeout(() => {
console.warn('Timeout on read video frame');
cleanup();
resolve();
}, 60000);
stream.on('data', handleChunk);
stream.on('end', onEnd);
stream.on('error', reject);
stream.resume();
}).then((data) => {
if (data) assert(data.length === targetSize);
function handleChunk(chunk) {
// console.log('chunk', chunk.length);
const nCopied = length + chunk.length > targetSize ? targetSize - length : chunk.length;
chunk.copy(buf, length, 0, nCopied);
length += nCopied;
if (length > targetSize) console.error('Video data overflow', length);
if (length >= targetSize) {
// console.log('Finished reading frame', inFrameCount, path);
const out = Buffer.from(buf);
const restLength = chunk.length - nCopied;
if (restLength > 0) {
// if (verbose) console.log('Left over data', nCopied, chunk.length, restLength);
chunk.slice(nCopied).copy(buf, 0);
length = restLength;
} else {
length = 0;
}
// inFrameCount += 1;
clearTimeout(timeout);
cleanup();
resolve(out);
}
}
timeout = setTimeout(() => {
console.warn('Timeout on read video frame');
cleanup();
resolve();
}, 60000);
stream.on('data', handleChunk);
stream.on('end', onEnd);
stream.on('error', reject);
stream.resume();
});
if (data) assert.equal(data.length, targetSize);
return data;
});
};
const close = () => {
if (verbose) console.log('Close', path);
@ -135,7 +170,7 @@ module.exports = async ({ width, height, channels, framerateStr, verbose, ffmpeg
};
return {
readNextFrame,
renderFrame,
close,
};
};
Write Preview
Loading…
Cancel
Save