需要有关代码的建议，因为它会杀死我的程序
Need recomendation about code, because it kill my programm
)
所以让它开始吧。我想实现下一个想法：我想使用 webrtc（交换视频和音频数据）与不同计算机上的其他用户联系，然后重新识别他的情绪。所以在这个项目中我使用 node-webrtc addon(here is examples)。所以我已经下载了示例并测试了视频合成示例，一切正常。
Here is result of testing
下一部分是识别面部表情。对于这个任务，我使用 face-api.js. I have tested this nice video。我不会附上照片，因为我现在使用 ubuntu，但在 windows 上测试过它，请相信我也一切正常。所以是时候将两个模块联合起来了。
作为主项目我使用的是node-webrtc示例，后续的讲解都会围绕这个模块展开。因此，要获得 运行 结果，您应该将 weights 文件夹从 face-api 复制到 node-webrtc/examples/video-compositing 文件夹中，然后只需替换下面的代码而不是 node-webrtc/example/video-compositing/server.js.
'use strict';
require('@tensorflow/tfjs-node');
const tf = require('@tensorflow/tfjs');
const nodeFetch = require('node-fetch');
const fapi = require('face-api.js');
const path = require('path');
const { createCanvas, createImageData } = require('canvas');
const { RTCVideoSink, RTCVideoSource, i420ToRgba, rgbaToI420 } = require('wrtc').nonstandard;
fapi.env.monkeyPatch({ fetch: nodeFetch });
const MODELS_URL = path.join(__dirname, '/weights');
const width = 640;
const height = 480;
Promise.all([
fapi.nets.tinyFaceDetector.loadFromDisk(MODELS_URL),
fapi.nets.faceLandmark68Net.loadFromDisk(MODELS_URL),
fapi.nets.faceRecognitionNet.loadFromDisk(MODELS_URL),
fapi.nets.faceExpressionNet.loadFromDisk(MODELS_URL)
]);
function beforeOffer(peerConnection) {
const source = new RTCVideoSource();
const track = source.createTrack();
const transceiver = peerConnection.addTransceiver(track);
const sink = new RTCVideoSink(transceiver.receiver.track);
let lastFrame = null;
function onFrame({ frame }) {
lastFrame = frame;
}
sink.addEventListener('frame', onFrame);
// TODO(mroberts): Is pixelFormat really necessary?
const canvas = createCanvas(width, height);
const context = canvas.getContext('2d', { pixelFormat: 'RGBA24' });
context.fillStyle = 'white';
context.fillRect(0, 0, width, height);
let emotion = '';
const interval = setInterval(() => {
if (lastFrame) {
const lastFrameCanvas = createCanvas(lastFrame.width, lastFrame.height);
const lastFrameContext = lastFrameCanvas.getContext('2d', { pixelFormat: 'RGBA24' });
const rgba = new Uint8ClampedArray(lastFrame.width * lastFrame.height * 4);
const rgbaFrame = createImageData(rgba, lastFrame.width, lastFrame.height);
i420ToRgba(lastFrame, rgbaFrame);
lastFrameContext.putImageData(rgbaFrame, 0, 0);
context.drawImage(lastFrameCanvas, 0, 0);
const emotionsArr = { 0: 'neutral', 1: 'happy', 2: 'sad', 3: 'angry', 4: 'fearful', 5: 'disgusted', 6: 'surprised' };
async function detectEmotion() {
let frameTensor3D = tf.browser.fromPixels(lastFrameCanvas)
let face = await fapi.detectSingleFace(frameTensor3D, new fapi.TinyFaceDetectorOptions()).withFaceExpressions();
//console.log(face);
function getEmotion(face) {
try {
let mostLikelyEmotion = emotionsArr[0];
let predictionArruracy = face.expressions[emotionsArr[0]];
for (let i = 0; i < Object.keys(face.expressions).length; i++) {
if (face.expressions[emotionsArr[i]] > predictionArruracy && face.expressions[emotionsArr[i]] < 1 ){
mostLikelyEmotion = emotionsArr[i];
predictionArruracy = face.expressions[emotionsArr[i]];
}
}
return mostLikelyEmotion;
}
catch (e){
return '';
}
}
let emot = getEmotion(face);
return emot;
}
detectEmotion().then(function(res) {
emotion = res;
});
} else {
context.fillStyle = 'rgba(255, 255, 255, 0.025)';
context.fillRect(0, 0, width, height);
}
if (emotion != ''){
context.font = '60px Sans-serif';
context.strokeStyle = 'black';
context.lineWidth = 1;
context.fillStyle = `rgba(${Math.round(255)}, ${Math.round(255)}, ${Math.round(255)}, 1)`;
context.textAlign = 'center';
context.save();
context.translate(width / 2, height);
context.strokeText(emotion, 0, 0);
context.fillText(emotion, 0, 0);
context.restore();
}
const rgbaFrame = context.getImageData(0, 0, width, height);
const i420Frame = {
width,
height,
data: new Uint8ClampedArray(1.5 * width * height)
};
rgbaToI420(rgbaFrame, i420Frame);
source.onFrame(i420Frame);
});
const { close } = peerConnection;
peerConnection.close = function() {
clearInterval(interval);
sink.stop();
track.stop();
return close.apply(this, arguments);
};
}
module.exports = { beforeOffer };
这里是 results1, result2 and result3 , all works fine))... Well, no, after 2-3 minutes my computer just stop do anything, i even cant move my mouse and then i get error "Killed" in terminal. I read about this error here，因为我只更改了项目中的一个脚本，我怀疑我的代码中的某处存在数据泄漏，并且我的 RAM 会随着时间的推移而变满。有人可以帮我解决这个问题吗？为什么程序以杀死进程结束？如果有人想自己测试它，我会留下包 json 以轻松安装所有要求。
{
"name": "node-webrtc-examples",
"version": "0.1.0",
"description": "This project presents a few example applications using node-webrtc.",
"private": true,
"main": "index.js",
"scripts": {
"lint": "eslint index.js examples lib test",
"start": "node index.js",
"test": "npm run test:unit && npm run test:integration",
"test:unit": "tape 'test/unit/**/*.js'",
"test:integration": "tape 'test/integration/**/*.js'"
},
"keywords": [
"Web",
"Audio"
],
"author": "Mark Andrus Roberts <markandrusroberts@gmail.com>",
"license": "BSD-3-Clause",
"dependencies": {
"@tensorflow/tfjs": "^1.2.9",
"@tensorflow/tfjs-core": "^1.2.9",
"@tensorflow/tfjs-node": "^1.2.9",
"Scope": "github:kevincennis/Scope",
"body-parser": "^1.18.3",
"browserify-middleware": "^8.1.1",
"canvas": "^2.6.0",
"color-space": "^1.16.0",
"express": "^4.16.4",
"face-api.js": "^0.21.0",
"node-fetch": "^2.3.0",
"uuid": "^3.3.2",
"wrtc": "^0.4.1"
},
"devDependencies": {
"eslint": "^5.15.1",
"tape": "^4.10.0"
}
}
如果你有像"someFunction is not a function"或类似的错误，那可能是因为你需要安装@tensorflow/tfjs-core、tfjs和tfjs-node 1.2.9版本。就像 npm i @tensorflow/tfjs-core@1.2.9。对于所有 3 个包。感谢您的回答和理解))
我在这一年里与 faceapi.js 和 tensorflow.js 一起工作，我测试了你的代码并且它没问题，但是在不到一分钟的时间内将我的 RAM 增加到 2GB，你有内存泄漏，使用时a Tensor 你应该释放内存吗？你好吗？
但是你应该在节点中使用 --inspect arg to inspect memory leak
只调用：
frameTensor3D.dispose();
我重构了你的代码，分享给你，希望对你有帮助：
"use strict";
require("@tensorflow/tfjs-node");
const tf = require("@tensorflow/tfjs");
const nodeFetch = require("node-fetch");
const fapi = require("face-api.js");
const path = require("path");
const { createCanvas, createImageData } = require("canvas");
const {
RTCVideoSink,
RTCVideoSource,
i420ToRgba,
rgbaToI420
} = require("wrtc").nonstandard;
fapi.env.monkeyPatch({ fetch: nodeFetch });
const MODELS_URL = path.join(__dirname, "/weights");
const width = 640;
const height = 480;
Promise.all([
fapi.nets.tinyFaceDetector.loadFromDisk(MODELS_URL),
fapi.nets.faceLandmark68Net.loadFromDisk(MODELS_URL),
fapi.nets.faceRecognitionNet.loadFromDisk(MODELS_URL),
fapi.nets.faceExpressionNet.loadFromDisk(MODELS_URL)
]);
function beforeOffer(peerConnection) {
const source = new RTCVideoSource();
const track = source.createTrack();
const transceiver = peerConnection.addTransceiver(track);
const sink = new RTCVideoSink(transceiver.receiver.track);
let lastFrame = null;
function onFrame({ frame }) {
lastFrame = frame;
}
sink.addEventListener("frame", onFrame);
// TODO(mroberts): Is pixelFormat really necessary?
const canvas = createCanvas(width, height);
const context = canvas.getContext("2d", { pixelFormat: "RGBA24" });
context.fillStyle = "white";
context.fillRect(0, 0, width, height);
const emotionsArr = {
0: "neutral",
1: "happy",
2: "sad",
3: "angry",
4: "fearful",
5: "disgusted",
6: "surprised"
};
async function detectEmotion(lastFrameCanvas) {
const frameTensor3D = tf.browser.fromPixels(lastFrameCanvas);
const face = await fapi
.detectSingleFace(
frameTensor3D,
new fapi.TinyFaceDetectorOptions({ inputSize: 160 })
)
.withFaceExpressions();
//console.log(face);
const emo = getEmotion(face);
frameTensor3D.dispose();
return emo;
}
function getEmotion(face) {
try {
let mostLikelyEmotion = emotionsArr[0];
let predictionArruracy = face.expressions[emotionsArr[0]];
for (let i = 0; i < Object.keys(face.expressions).length; i++) {
if (
face.expressions[emotionsArr[i]] > predictionArruracy &&
face.expressions[emotionsArr[i]] < 1
) {
mostLikelyEmotion = emotionsArr[i];
predictionArruracy = face.expressions[emotionsArr[i]];
}
}
//console.log(mostLikelyEmotion);
return mostLikelyEmotion;
} catch (e) {
return "";
}
}
let emotion = "";
const interval = setInterval(() => {
if (lastFrame) {
const lastFrameCanvas = createCanvas(lastFrame.width, lastFrame.height);
const lastFrameContext = lastFrameCanvas.getContext("2d", {
pixelFormat: "RGBA24"
});
const rgba = new Uint8ClampedArray(
lastFrame.width * lastFrame.height * 4
);
const rgbaFrame = createImageData(
rgba,
lastFrame.width,
lastFrame.height
);
i420ToRgba(lastFrame, rgbaFrame);
lastFrameContext.putImageData(rgbaFrame, 0, 0);
context.drawImage(lastFrameCanvas, 0, 0);
detectEmotion(lastFrameCanvas).then(function(res) {
emotion = res;
});
} else {
context.fillStyle = "rgba(255, 255, 255, 0.025)";
context.fillRect(0, 0, width, height);
}
if (emotion != "") {
context.font = "60px Sans-serif";
context.strokeStyle = "black";
context.lineWidth = 1;
context.fillStyle = `rgba(${Math.round(255)}, ${Math.round(
255
)}, ${Math.round(255)}, 1)`;
context.textAlign = "center";
context.save();
context.translate(width / 2, height);
context.strokeText(emotion, 0, 0);
context.fillText(emotion, 0, 0);
context.restore();
}
const rgbaFrame = context.getImageData(0, 0, width, height);
const i420Frame = {
width,
height,
data: new Uint8ClampedArray(1.5 * width * height)
};
rgbaToI420(rgbaFrame, i420Frame);
source.onFrame(i420Frame);
});
const { close } = peerConnection;
peerConnection.close = function() {
clearInterval(interval);
sink.stop();
track.stop();
return close.apply(this, arguments);
};
}
module.exports = { beforeOffer };
对不起我的英语，祝你好运
) 所以让它开始吧。我想实现下一个想法：我想使用 webrtc（交换视频和音频数据）与不同计算机上的其他用户联系，然后重新识别他的情绪。所以在这个项目中我使用 node-webrtc addon(here is examples)。所以我已经下载了示例并测试了视频合成示例，一切正常。 Here is result of testing
下一部分是识别面部表情。对于这个任务，我使用 face-api.js. I have tested this nice video。我不会附上照片，因为我现在使用 ubuntu，但在 windows 上测试过它，请相信我也一切正常。所以是时候将两个模块联合起来了。
作为主项目我使用的是node-webrtc示例，后续的讲解都会围绕这个模块展开。因此，要获得 运行 结果，您应该将 weights 文件夹从 face-api 复制到 node-webrtc/examples/video-compositing 文件夹中，然后只需替换下面的代码而不是 node-webrtc/example/video-compositing/server.js.
'use strict';
require('@tensorflow/tfjs-node');
const tf = require('@tensorflow/tfjs');
const nodeFetch = require('node-fetch');
const fapi = require('face-api.js');
const path = require('path');
const { createCanvas, createImageData } = require('canvas');
const { RTCVideoSink, RTCVideoSource, i420ToRgba, rgbaToI420 } = require('wrtc').nonstandard;
fapi.env.monkeyPatch({ fetch: nodeFetch });
const MODELS_URL = path.join(__dirname, '/weights');
const width = 640;
const height = 480;
Promise.all([
fapi.nets.tinyFaceDetector.loadFromDisk(MODELS_URL),
fapi.nets.faceLandmark68Net.loadFromDisk(MODELS_URL),
fapi.nets.faceRecognitionNet.loadFromDisk(MODELS_URL),
fapi.nets.faceExpressionNet.loadFromDisk(MODELS_URL)
]);
function beforeOffer(peerConnection) {
const source = new RTCVideoSource();
const track = source.createTrack();
const transceiver = peerConnection.addTransceiver(track);
const sink = new RTCVideoSink(transceiver.receiver.track);
let lastFrame = null;
function onFrame({ frame }) {
lastFrame = frame;
}
sink.addEventListener('frame', onFrame);
// TODO(mroberts): Is pixelFormat really necessary?
const canvas = createCanvas(width, height);
const context = canvas.getContext('2d', { pixelFormat: 'RGBA24' });
context.fillStyle = 'white';
context.fillRect(0, 0, width, height);
let emotion = '';
const interval = setInterval(() => {
if (lastFrame) {
const lastFrameCanvas = createCanvas(lastFrame.width, lastFrame.height);
const lastFrameContext = lastFrameCanvas.getContext('2d', { pixelFormat: 'RGBA24' });
const rgba = new Uint8ClampedArray(lastFrame.width * lastFrame.height * 4);
const rgbaFrame = createImageData(rgba, lastFrame.width, lastFrame.height);
i420ToRgba(lastFrame, rgbaFrame);
lastFrameContext.putImageData(rgbaFrame, 0, 0);
context.drawImage(lastFrameCanvas, 0, 0);
const emotionsArr = { 0: 'neutral', 1: 'happy', 2: 'sad', 3: 'angry', 4: 'fearful', 5: 'disgusted', 6: 'surprised' };
async function detectEmotion() {
let frameTensor3D = tf.browser.fromPixels(lastFrameCanvas)
let face = await fapi.detectSingleFace(frameTensor3D, new fapi.TinyFaceDetectorOptions()).withFaceExpressions();
//console.log(face);
function getEmotion(face) {
try {
let mostLikelyEmotion = emotionsArr[0];
let predictionArruracy = face.expressions[emotionsArr[0]];
for (let i = 0; i < Object.keys(face.expressions).length; i++) {
if (face.expressions[emotionsArr[i]] > predictionArruracy && face.expressions[emotionsArr[i]] < 1 ){
mostLikelyEmotion = emotionsArr[i];
predictionArruracy = face.expressions[emotionsArr[i]];
}
}
return mostLikelyEmotion;
}
catch (e){
return '';
}
}
let emot = getEmotion(face);
return emot;
}
detectEmotion().then(function(res) {
emotion = res;
});
} else {
context.fillStyle = 'rgba(255, 255, 255, 0.025)';
context.fillRect(0, 0, width, height);
}
if (emotion != ''){
context.font = '60px Sans-serif';
context.strokeStyle = 'black';
context.lineWidth = 1;
context.fillStyle = `rgba(${Math.round(255)}, ${Math.round(255)}, ${Math.round(255)}, 1)`;
context.textAlign = 'center';
context.save();
context.translate(width / 2, height);
context.strokeText(emotion, 0, 0);
context.fillText(emotion, 0, 0);
context.restore();
}
const rgbaFrame = context.getImageData(0, 0, width, height);
const i420Frame = {
width,
height,
data: new Uint8ClampedArray(1.5 * width * height)
};
rgbaToI420(rgbaFrame, i420Frame);
source.onFrame(i420Frame);
});
const { close } = peerConnection;
peerConnection.close = function() {
clearInterval(interval);
sink.stop();
track.stop();
return close.apply(this, arguments);
};
}
module.exports = { beforeOffer };
这里是 results1, result2 and result3 , all works fine))... Well, no, after 2-3 minutes my computer just stop do anything, i even cant move my mouse and then i get error "Killed" in terminal. I read about this error here，因为我只更改了项目中的一个脚本，我怀疑我的代码中的某处存在数据泄漏，并且我的 RAM 会随着时间的推移而变满。有人可以帮我解决这个问题吗？为什么程序以杀死进程结束？如果有人想自己测试它，我会留下包 json 以轻松安装所有要求。
{
"name": "node-webrtc-examples",
"version": "0.1.0",
"description": "This project presents a few example applications using node-webrtc.",
"private": true,
"main": "index.js",
"scripts": {
"lint": "eslint index.js examples lib test",
"start": "node index.js",
"test": "npm run test:unit && npm run test:integration",
"test:unit": "tape 'test/unit/**/*.js'",
"test:integration": "tape 'test/integration/**/*.js'"
},
"keywords": [
"Web",
"Audio"
],
"author": "Mark Andrus Roberts <markandrusroberts@gmail.com>",
"license": "BSD-3-Clause",
"dependencies": {
"@tensorflow/tfjs": "^1.2.9",
"@tensorflow/tfjs-core": "^1.2.9",
"@tensorflow/tfjs-node": "^1.2.9",
"Scope": "github:kevincennis/Scope",
"body-parser": "^1.18.3",
"browserify-middleware": "^8.1.1",
"canvas": "^2.6.0",
"color-space": "^1.16.0",
"express": "^4.16.4",
"face-api.js": "^0.21.0",
"node-fetch": "^2.3.0",
"uuid": "^3.3.2",
"wrtc": "^0.4.1"
},
"devDependencies": {
"eslint": "^5.15.1",
"tape": "^4.10.0"
}
}
如果你有像"someFunction is not a function"或类似的错误，那可能是因为你需要安装@tensorflow/tfjs-core、tfjs和tfjs-node 1.2.9版本。就像 npm i @tensorflow/tfjs-core@1.2.9。对于所有 3 个包。感谢您的回答和理解))
我在这一年里与 faceapi.js 和 tensorflow.js 一起工作，我测试了你的代码并且它没问题，但是在不到一分钟的时间内将我的 RAM 增加到 2GB，你有内存泄漏，使用时a Tensor 你应该释放内存吗？你好吗？
但是你应该在节点中使用 --inspect arg to inspect memory leak
只调用：
frameTensor3D.dispose();
我重构了你的代码，分享给你，希望对你有帮助：
"use strict";
require("@tensorflow/tfjs-node");
const tf = require("@tensorflow/tfjs");
const nodeFetch = require("node-fetch");
const fapi = require("face-api.js");
const path = require("path");
const { createCanvas, createImageData } = require("canvas");
const {
RTCVideoSink,
RTCVideoSource,
i420ToRgba,
rgbaToI420
} = require("wrtc").nonstandard;
fapi.env.monkeyPatch({ fetch: nodeFetch });
const MODELS_URL = path.join(__dirname, "/weights");
const width = 640;
const height = 480;
Promise.all([
fapi.nets.tinyFaceDetector.loadFromDisk(MODELS_URL),
fapi.nets.faceLandmark68Net.loadFromDisk(MODELS_URL),
fapi.nets.faceRecognitionNet.loadFromDisk(MODELS_URL),
fapi.nets.faceExpressionNet.loadFromDisk(MODELS_URL)
]);
function beforeOffer(peerConnection) {
const source = new RTCVideoSource();
const track = source.createTrack();
const transceiver = peerConnection.addTransceiver(track);
const sink = new RTCVideoSink(transceiver.receiver.track);
let lastFrame = null;
function onFrame({ frame }) {
lastFrame = frame;
}
sink.addEventListener("frame", onFrame);
// TODO(mroberts): Is pixelFormat really necessary?
const canvas = createCanvas(width, height);
const context = canvas.getContext("2d", { pixelFormat: "RGBA24" });
context.fillStyle = "white";
context.fillRect(0, 0, width, height);
const emotionsArr = {
0: "neutral",
1: "happy",
2: "sad",
3: "angry",
4: "fearful",
5: "disgusted",
6: "surprised"
};
async function detectEmotion(lastFrameCanvas) {
const frameTensor3D = tf.browser.fromPixels(lastFrameCanvas);
const face = await fapi
.detectSingleFace(
frameTensor3D,
new fapi.TinyFaceDetectorOptions({ inputSize: 160 })
)
.withFaceExpressions();
//console.log(face);
const emo = getEmotion(face);
frameTensor3D.dispose();
return emo;
}
function getEmotion(face) {
try {
let mostLikelyEmotion = emotionsArr[0];
let predictionArruracy = face.expressions[emotionsArr[0]];
for (let i = 0; i < Object.keys(face.expressions).length; i++) {
if (
face.expressions[emotionsArr[i]] > predictionArruracy &&
face.expressions[emotionsArr[i]] < 1
) {
mostLikelyEmotion = emotionsArr[i];
predictionArruracy = face.expressions[emotionsArr[i]];
}
}
//console.log(mostLikelyEmotion);
return mostLikelyEmotion;
} catch (e) {
return "";
}
}
let emotion = "";
const interval = setInterval(() => {
if (lastFrame) {
const lastFrameCanvas = createCanvas(lastFrame.width, lastFrame.height);
const lastFrameContext = lastFrameCanvas.getContext("2d", {
pixelFormat: "RGBA24"
});
const rgba = new Uint8ClampedArray(
lastFrame.width * lastFrame.height * 4
);
const rgbaFrame = createImageData(
rgba,
lastFrame.width,
lastFrame.height
);
i420ToRgba(lastFrame, rgbaFrame);
lastFrameContext.putImageData(rgbaFrame, 0, 0);
context.drawImage(lastFrameCanvas, 0, 0);
detectEmotion(lastFrameCanvas).then(function(res) {
emotion = res;
});
} else {
context.fillStyle = "rgba(255, 255, 255, 0.025)";
context.fillRect(0, 0, width, height);
}
if (emotion != "") {
context.font = "60px Sans-serif";
context.strokeStyle = "black";
context.lineWidth = 1;
context.fillStyle = `rgba(${Math.round(255)}, ${Math.round(
255
)}, ${Math.round(255)}, 1)`;
context.textAlign = "center";
context.save();
context.translate(width / 2, height);
context.strokeText(emotion, 0, 0);
context.fillText(emotion, 0, 0);
context.restore();
}
const rgbaFrame = context.getImageData(0, 0, width, height);
const i420Frame = {
width,
height,
data: new Uint8ClampedArray(1.5 * width * height)
};
rgbaToI420(rgbaFrame, i420Frame);
source.onFrame(i420Frame);
});
const { close } = peerConnection;
peerConnection.close = function() {
clearInterval(interval);
sink.stop();
track.stop();
return close.apply(this, arguments);
};
}
module.exports = { beforeOffer };
对不起我的英语，祝你好运