使用 Mediapipe 的自分割 javascript 将用户自拍作为纹理传递到 Networked-Aframe 上以实现多人游戏体验
Using Self-Segmentation javascript of Mediapipe to pass user selfie as a texture on Networked-Aframe for multiplaying experiences
好吧，我的目标是制作一个多人游戏 3D 环境，其中人物由立方体表示，这些立方体的纹理是没有背景的自拍。它是一种没有色度键背景的廉价虚拟制作。使用 MediaPipe Selfie-Segmentation 移除背景。问题是，不是在立方体上有其他玩家纹理（P1 应该看到 P2，P2 应该看到 P1，每个人都看到他的自拍。这意味着 P1 看到 P1，P2 看到 P2，这是不好的。
现场演示：https://vrodos-multiplaying.iti.gr/plain_aframe_mediapipe_testbed.html
说明：您应该使用台式机、笔记本电脑或手机两台机器进行测试。仅使用 Chrome 因为 Mediapipe 在其他浏览器上不工作。如果网页卡住，请重新加载网页（Mediapipe 有时会很粘）。至少需要两台机器加载网页才能启动多人游戏环境。
代码：
<html>
<head>
<script src="https://aframe.io/releases/1.2.0/aframe.min.js"></script>
<!-- Selfie Segmentation of Mediapipe -->
<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils@0.6/control_utils.css" crossorigin="anonymous">
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils@0.3/camera_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils@0.6/control_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils@0.3/drawing_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/selfie_segmentation.js" crossorigin="anonymous"></script>
<!-- Networked A-frame -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.3.0/socket.io.slim.js"></script>
<script src="/easyrtc/easyrtc.js"></script>
<script src="https://unpkg.com/networked-aframe/dist/networked-aframe.min.js"></script>
</head>
<body>
<a-scene networked-scene="
adapter: easyrtc;
video: true;
debug: true;
connectOnLoad: true;">
<a-assets>
<template id="avatar-template">
<a-box material="alphaTest: 0.5; transparent: true; side: both;"
width="1"
height="1"
position="0 0 0" rotation="0 0 0"
networked-video-source-mediapiped></a-box>
</template>
</a-assets>
<a-entity id="player"
networked="template:#avatar-template;attachTemplateToLocal:false;"
camera wasd-controls look-controls>
</a-entity>
<a-plane position="0 -2 -4" rotation="-90 0 0" width="4" height="4" color="#7BC8A4" ></a-plane>
<a-sky color="#777"></a-sky>
</a-scene>
</body>
<script>
// Networked Aframe : Register new component for streaming the Selfie-Segmentation video stream
AFRAME.registerComponent('networked-video-source-mediapiped', {
schema: {
},
dependencies: ['material'],
init: function () {
this.videoTexture = null;
this.video = null;
this.stream = null;
this._setMediaStream = this._setMediaStream.bind(this);
NAF.utils.getNetworkedEntity(this.el).then((networkedEl) => {
const ownerId = networkedEl.components.networked.data.owner;
if (ownerId) {
NAF.connection.adapter.getMediaStream(ownerId, "video")
.then(this._setMediaStream)
.catch((e) => NAF.log.error(`Error getting media stream for ${ownerId}`, e));
} else {
// Correctly configured local entity, perhaps do something here for enabling debug audio loopback
}
});
},
_setMediaStream(newStream) {
if(!this.video) {
this.setupVideo();
}
if(newStream != this.stream) {
if (this.stream) {
this._clearMediaStream();
}
if (newStream) {
this.video.srcObject = canvasElement.captureStream(30);
this.videoTexture = new THREE.VideoTexture(this.video);
this.videoTexture.format = THREE.RGBAFormat;
// Mesh to send
const mesh = this.el.getObject3D('mesh');
mesh.material.map = this.videoTexture;
mesh.material.needsUpdate = true;
}
this.stream = newStream;
}
},
_clearMediaStream() {
this.stream = null;
if (this.videoTexture) {
if (this.videoTexture.image instanceof HTMLVideoElement) {
// Note: this.videoTexture.image === this.video
const video = this.videoTexture.image;
video.pause();
video.srcObject = null;
video.load();
}
this.videoTexture.dispose();
this.videoTexture = null;
}
},
remove: function() {
this._clearMediaStream();
},
setupVideo: function() {
if (!this.video) {
const video = document.createElement('video');
video.setAttribute('autoplay', true);
video.setAttribute('playsinline', true);
video.setAttribute('muted', true);
this.video = video;
}
}
});
// ----- Mediapipe ------
const controls = window;
//const mpSelfieSegmentation = window;
const examples = {
images: [],
// {name: 'name', src: 'https://url.com'},
videos: [],
};
const fpsControl = new controls.FPS();
let activeEffect = 'background';
const controlsElement = document.createElement('control-panel');
var canvasElement = document.createElement('canvas');
canvasElement.height= 1000;
canvasElement.width = 1000;
var canvasCtx = canvasElement.getContext('2d');
// --------
function drawResults(results) {
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
canvasCtx.drawImage(results.segmentationMask, 0, 0, canvasElement.width, canvasElement.height);
canvasCtx.globalCompositeOperation = 'source-in';
canvasCtx.drawImage(results.image, 0, 0, canvasElement.width, canvasElement.height);
}
const selfieSegmentation = new SelfieSegmentation({
locateFile: (file) => {
console.log(file);
return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${file}`;
}
});
selfieSegmentation.onResults(drawResults);
// -------------
new controls
.ControlPanel(controlsElement, {
selfieMode: true,
modelSelection: 1,
effect: 'background',
})
.add([
new controls.StaticText({title: 'MediaPipe Selfie Segmentation'}),
fpsControl,
new controls.Toggle({title: 'Selfie Mode', field: 'selfieMode'}),
new controls.SourcePicker({
onSourceChanged: () => {
selfieSegmentation.reset();
},
onFrame: async (input, size) => {
const aspect = size.height / size.width;
let width, height;
if (window.innerWidth > window.innerHeight) {
height = window.innerHeight;
width = height / aspect;
} else {
width = window.innerWidth;
height = width * aspect;
}
canvasElement.width = width;
canvasElement.height = height;
await selfieSegmentation.send({image: input});
},
examples: examples
}),
new controls.Slider({
title: 'Model Selection',
field: 'modelSelection',
discrete: ['General', 'Landscape'],
}),
new controls.Slider({
title: 'Effect',
field: 'effect',
discrete: {'background': 'Background', 'mask': 'Foreground'},
}),
])
.on(x => {
const options = x;
//videoElement.classList.toggle('selfie', options.selfieMode);
activeEffect = x['effect'];
selfieSegmentation.setOptions(options);
});
</script>
</html>
截图：
此处播放器 1 看到播放器 1 流而不是播放器 2 流 (Grrrrrr)：
嗯，我找到了。问题是 MediaStream 可以有很多视频轨道。在这里查看我的回答：
https://github.com/networked-aframe/networked-aframe/issues/269
不幸的是，networked-aframe EasyRtcAdapter 不支持很多 MediaStreams，但是很容易添加另一个视频轨道，然后获取 videotrack[1] 而不是 videotrack[0]。我应该制作一个特殊的 EasyRtcAdapter 以避免有两个视频轨道并避免带宽压力过大。
好吧，我的目标是制作一个多人游戏 3D 环境，其中人物由立方体表示，这些立方体的纹理是没有背景的自拍。它是一种没有色度键背景的廉价虚拟制作。使用 MediaPipe Selfie-Segmentation 移除背景。问题是，不是在立方体上有其他玩家纹理（P1 应该看到 P2，P2 应该看到 P1，每个人都看到他的自拍。这意味着 P1 看到 P1，P2 看到 P2，这是不好的。
现场演示：https://vrodos-multiplaying.iti.gr/plain_aframe_mediapipe_testbed.html
说明：您应该使用台式机、笔记本电脑或手机两台机器进行测试。仅使用 Chrome 因为 Mediapipe 在其他浏览器上不工作。如果网页卡住，请重新加载网页（Mediapipe 有时会很粘）。至少需要两台机器加载网页才能启动多人游戏环境。
代码：
<html>
<head>
<script src="https://aframe.io/releases/1.2.0/aframe.min.js"></script>
<!-- Selfie Segmentation of Mediapipe -->
<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils@0.6/control_utils.css" crossorigin="anonymous">
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils@0.3/camera_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils@0.6/control_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils@0.3/drawing_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/selfie_segmentation.js" crossorigin="anonymous"></script>
<!-- Networked A-frame -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.3.0/socket.io.slim.js"></script>
<script src="/easyrtc/easyrtc.js"></script>
<script src="https://unpkg.com/networked-aframe/dist/networked-aframe.min.js"></script>
</head>
<body>
<a-scene networked-scene="
adapter: easyrtc;
video: true;
debug: true;
connectOnLoad: true;">
<a-assets>
<template id="avatar-template">
<a-box material="alphaTest: 0.5; transparent: true; side: both;"
width="1"
height="1"
position="0 0 0" rotation="0 0 0"
networked-video-source-mediapiped></a-box>
</template>
</a-assets>
<a-entity id="player"
networked="template:#avatar-template;attachTemplateToLocal:false;"
camera wasd-controls look-controls>
</a-entity>
<a-plane position="0 -2 -4" rotation="-90 0 0" width="4" height="4" color="#7BC8A4" ></a-plane>
<a-sky color="#777"></a-sky>
</a-scene>
</body>
<script>
// Networked Aframe : Register new component for streaming the Selfie-Segmentation video stream
AFRAME.registerComponent('networked-video-source-mediapiped', {
schema: {
},
dependencies: ['material'],
init: function () {
this.videoTexture = null;
this.video = null;
this.stream = null;
this._setMediaStream = this._setMediaStream.bind(this);
NAF.utils.getNetworkedEntity(this.el).then((networkedEl) => {
const ownerId = networkedEl.components.networked.data.owner;
if (ownerId) {
NAF.connection.adapter.getMediaStream(ownerId, "video")
.then(this._setMediaStream)
.catch((e) => NAF.log.error(`Error getting media stream for ${ownerId}`, e));
} else {
// Correctly configured local entity, perhaps do something here for enabling debug audio loopback
}
});
},
_setMediaStream(newStream) {
if(!this.video) {
this.setupVideo();
}
if(newStream != this.stream) {
if (this.stream) {
this._clearMediaStream();
}
if (newStream) {
this.video.srcObject = canvasElement.captureStream(30);
this.videoTexture = new THREE.VideoTexture(this.video);
this.videoTexture.format = THREE.RGBAFormat;
// Mesh to send
const mesh = this.el.getObject3D('mesh');
mesh.material.map = this.videoTexture;
mesh.material.needsUpdate = true;
}
this.stream = newStream;
}
},
_clearMediaStream() {
this.stream = null;
if (this.videoTexture) {
if (this.videoTexture.image instanceof HTMLVideoElement) {
// Note: this.videoTexture.image === this.video
const video = this.videoTexture.image;
video.pause();
video.srcObject = null;
video.load();
}
this.videoTexture.dispose();
this.videoTexture = null;
}
},
remove: function() {
this._clearMediaStream();
},
setupVideo: function() {
if (!this.video) {
const video = document.createElement('video');
video.setAttribute('autoplay', true);
video.setAttribute('playsinline', true);
video.setAttribute('muted', true);
this.video = video;
}
}
});
// ----- Mediapipe ------
const controls = window;
//const mpSelfieSegmentation = window;
const examples = {
images: [],
// {name: 'name', src: 'https://url.com'},
videos: [],
};
const fpsControl = new controls.FPS();
let activeEffect = 'background';
const controlsElement = document.createElement('control-panel');
var canvasElement = document.createElement('canvas');
canvasElement.height= 1000;
canvasElement.width = 1000;
var canvasCtx = canvasElement.getContext('2d');
// --------
function drawResults(results) {
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
canvasCtx.drawImage(results.segmentationMask, 0, 0, canvasElement.width, canvasElement.height);
canvasCtx.globalCompositeOperation = 'source-in';
canvasCtx.drawImage(results.image, 0, 0, canvasElement.width, canvasElement.height);
}
const selfieSegmentation = new SelfieSegmentation({
locateFile: (file) => {
console.log(file);
return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${file}`;
}
});
selfieSegmentation.onResults(drawResults);
// -------------
new controls
.ControlPanel(controlsElement, {
selfieMode: true,
modelSelection: 1,
effect: 'background',
})
.add([
new controls.StaticText({title: 'MediaPipe Selfie Segmentation'}),
fpsControl,
new controls.Toggle({title: 'Selfie Mode', field: 'selfieMode'}),
new controls.SourcePicker({
onSourceChanged: () => {
selfieSegmentation.reset();
},
onFrame: async (input, size) => {
const aspect = size.height / size.width;
let width, height;
if (window.innerWidth > window.innerHeight) {
height = window.innerHeight;
width = height / aspect;
} else {
width = window.innerWidth;
height = width * aspect;
}
canvasElement.width = width;
canvasElement.height = height;
await selfieSegmentation.send({image: input});
},
examples: examples
}),
new controls.Slider({
title: 'Model Selection',
field: 'modelSelection',
discrete: ['General', 'Landscape'],
}),
new controls.Slider({
title: 'Effect',
field: 'effect',
discrete: {'background': 'Background', 'mask': 'Foreground'},
}),
])
.on(x => {
const options = x;
//videoElement.classList.toggle('selfie', options.selfieMode);
activeEffect = x['effect'];
selfieSegmentation.setOptions(options);
});
</script>
</html>
截图：
此处播放器 1 看到播放器 1 流而不是播放器 2 流 (Grrrrrr)：
嗯，我找到了。问题是 MediaStream 可以有很多视频轨道。在这里查看我的回答：
https://github.com/networked-aframe/networked-aframe/issues/269
不幸的是，networked-aframe EasyRtcAdapter 不支持很多 MediaStreams，但是很容易添加另一个视频轨道，然后获取 videotrack[1] 而不是 videotrack[0]。我应该制作一个特殊的 EasyRtcAdapter 以避免有两个视频轨道并避免带宽压力过大。