如何使用Nodejs将视频帧直接读入内存?
how to read video Frames directly into memory with Nodejs?
我想做的是拍摄一段视频并将其分解为帧,然后将这些帧传递给模型以检测每一帧中的对象,但问题是提取过程花费了太多时间,我不需要我磁盘上的帧。
fmpeg-stream
提供流功能。所以不需要写入文件。
也可以直接使用ffmpeg
和spawn
新的子进程。它的 .stdout
属性 是一个可读流。在event
数据上,可以读取chunk。
const fs = require("fs");
const tf = require("@tensorflow/tfjs-node")
const logStream = fs.createWriteStream('./logFile.log');
const spawnProcess = require('child_process').spawn,
ffmpeg = spawnProcess('ffmpeg', [
'-i', 'videfile.mp4',
'-vcodec', 'png',
'-f', 'rawvideo',
'-s', 'h*w', // size of one frame
'pipe:1'
]);
ffmpeg.stderr.pipe(logStream); // for debugging
let i = 0
ffmpeg.stdout.on('data', (data) => {
try {
console.log(tf.node.decodeImage(data).shape)
console.log(`${++i} frames read`)
// dispose all tensors
} catch(e) {
console.log(e)
}
})
ffmpeg.on('close', function (code) {
console.log('child process exited with code ' + code);
});
解码图像在 try catch
块中,以防止当块与帧不匹配时引发错误。
防止解码与图像不对应的块的更健壮的代码如下:
const { Transform } = require("stream")
class ExtractFrames extends Transform {
constructor(delimiter) {
super({ readableObjectMode: true })
this.delimiter = Buffer.from(delimiter, "hex")
this.buffer = Buffer.alloc(0)
}
_transform(data, enc, cb) {
// Add new data to buffer
this.buffer = Buffer.concat([this.buffer, data])
const start = this.buffer.indexOf(this.delimiter)
if (start < 0) return // there's no frame data at all
const end = this.buffer.indexOf(
this.delimiter,
start + this.delimiter.length,
)
if (end < 0) return // we haven't got the whole frame yet
this.push(this.buffer.slice(start, end)) // emit a frame
this.buffer = this.buffer.slice(end) // remove frame data from buffer
if (start > 0) console.error(`Discarded ${start} bytes of invalid data`)
cb()
}
_flush(callback) {
// push remaining buffer to readable stream
callback(null, this.buffer);
}
}
const fs = require("fs");
const tf = require("@tensorflow/tfjs-node")
const logStream = fs.createWriteStream('./logFile.log');
const spawnProcess = require('child_process').spawn,
ffmpeg = spawnProcess('ffmpeg', [
'-i', 'generique.mp4',
'-vcodec', 'mjpeg',
'-f', 'rawvideo',
'-s', '420x360', // size of one frame
'pipe:1'
]);
ffmpeg.stderr.pipe(logStream); // for debugging
let i = 0
ffmpeg.stdout
.pipe(new ExtractFrames("FFD8FF")).on('data', (data) => {
try {
console.log(tf.node.decodeImage(data).shape)
console.log(`${++i} frames read`)
// dispose all tensors
} catch(e) {
console.log(e)
}
})
ffmpeg.on('close', function (code) {
console.log('child process exited with code ' + code);
});
尽管上面的代码有效,它仍然会很快填满内存。将帧提取与数据处理本身分开会有所帮助。
async function* frames() {
let resolve;
let promise = new Promise(r => resolve = r);
let bool = true;
ls.stdout.pipe(new ExtractFrames("FFD8FF")).on('data', data => {
resolve(data);
promise = new Promise(r => resolve = r);
});
ls.on('close', function (code) {
bool = false
console.log('code')
});
while (bool) {
const data = await promise;
yield data;
}
}
(async() => {
// data processing
// possibly create tf.dataset for training
for await (const data of stream()) {
console.log(tf.node.decodeImage(data).shape)
console.log(data);
}
})()
我想做的是拍摄一段视频并将其分解为帧,然后将这些帧传递给模型以检测每一帧中的对象,但问题是提取过程花费了太多时间,我不需要我磁盘上的帧。
fmpeg-stream
提供流功能。所以不需要写入文件。
也可以直接使用ffmpeg
和spawn
新的子进程。它的 .stdout
属性 是一个可读流。在event
数据上,可以读取chunk。
const fs = require("fs");
const tf = require("@tensorflow/tfjs-node")
const logStream = fs.createWriteStream('./logFile.log');
const spawnProcess = require('child_process').spawn,
ffmpeg = spawnProcess('ffmpeg', [
'-i', 'videfile.mp4',
'-vcodec', 'png',
'-f', 'rawvideo',
'-s', 'h*w', // size of one frame
'pipe:1'
]);
ffmpeg.stderr.pipe(logStream); // for debugging
let i = 0
ffmpeg.stdout.on('data', (data) => {
try {
console.log(tf.node.decodeImage(data).shape)
console.log(`${++i} frames read`)
// dispose all tensors
} catch(e) {
console.log(e)
}
})
ffmpeg.on('close', function (code) {
console.log('child process exited with code ' + code);
});
解码图像在 try catch
块中,以防止当块与帧不匹配时引发错误。
防止解码与图像不对应的块的更健壮的代码如下:
const { Transform } = require("stream")
class ExtractFrames extends Transform {
constructor(delimiter) {
super({ readableObjectMode: true })
this.delimiter = Buffer.from(delimiter, "hex")
this.buffer = Buffer.alloc(0)
}
_transform(data, enc, cb) {
// Add new data to buffer
this.buffer = Buffer.concat([this.buffer, data])
const start = this.buffer.indexOf(this.delimiter)
if (start < 0) return // there's no frame data at all
const end = this.buffer.indexOf(
this.delimiter,
start + this.delimiter.length,
)
if (end < 0) return // we haven't got the whole frame yet
this.push(this.buffer.slice(start, end)) // emit a frame
this.buffer = this.buffer.slice(end) // remove frame data from buffer
if (start > 0) console.error(`Discarded ${start} bytes of invalid data`)
cb()
}
_flush(callback) {
// push remaining buffer to readable stream
callback(null, this.buffer);
}
}
const fs = require("fs");
const tf = require("@tensorflow/tfjs-node")
const logStream = fs.createWriteStream('./logFile.log');
const spawnProcess = require('child_process').spawn,
ffmpeg = spawnProcess('ffmpeg', [
'-i', 'generique.mp4',
'-vcodec', 'mjpeg',
'-f', 'rawvideo',
'-s', '420x360', // size of one frame
'pipe:1'
]);
ffmpeg.stderr.pipe(logStream); // for debugging
let i = 0
ffmpeg.stdout
.pipe(new ExtractFrames("FFD8FF")).on('data', (data) => {
try {
console.log(tf.node.decodeImage(data).shape)
console.log(`${++i} frames read`)
// dispose all tensors
} catch(e) {
console.log(e)
}
})
ffmpeg.on('close', function (code) {
console.log('child process exited with code ' + code);
});
尽管上面的代码有效,它仍然会很快填满内存。将帧提取与数据处理本身分开会有所帮助。
async function* frames() {
let resolve;
let promise = new Promise(r => resolve = r);
let bool = true;
ls.stdout.pipe(new ExtractFrames("FFD8FF")).on('data', data => {
resolve(data);
promise = new Promise(r => resolve = r);
});
ls.on('close', function (code) {
bool = false
console.log('code')
});
while (bool) {
const data = await promise;
yield data;
}
}
(async() => {
// data processing
// possibly create tf.dataset for training
for await (const data of stream()) {
console.log(tf.node.decodeImage(data).shape)
console.log(data);
}
})()