使用 Canvas 的视频作为视频的遮罩
Using Video as Mask of Video with Canvas
我一直在尝试创建一个从一个视频到另一个视频的动态遮罩。这两个视频都没有 alpha 通道（即便如此，现在浏览器对它的支持也很低），所以我试图用 canvas 解决这个问题。我设法在非常有限的周期内完成了它，但它仍然无法在我的输出中以合适的帧速率完成此操作（它达到大约 20 fps，并且至少应该达到 25/30 才能显得平滑）。它实际上是 运行 的单数形式，周围有其他元素，所以我需要提高它的效率。
由于保密原因，我不能真正分享实际的视频，但其中一个包含黑色和白色的 alpha 掩码 (mask.mp4)（以及介于两者之间的灰色阴影），而另一个视频内容可以是任何需要屏蔽的内容 (video.mp4).
我要改进的代码实际上是这样的（假设所有视频和canvas元素都是1920x1080）：
const video = document.getElementById( 'video' ); // `<video src="video.mp4"></video>
const mask = document.getElementById( 'mask' ); // `<video src="mask.mp4"></video>
const buffer = document.getElementById( 'buffer' ); // <canvas hidden></canvas>
const bufferCtx = buffer.getContext( '2d' );
const output = document.getElementById( 'output' ); // <canvas></canvas>
const outputCtx = output.getContext( '2d' );
outputCtx.globalCompositeOperation = 'source-in';
function maskVideo(){
// Draw the mask
bufferCtx.drawImage( mask, 0, 0 );
// Get image data
const data = bufferCtx.getImageData( 0, 0, w, h );
// Assign the grayscale value to the alpha in the image data
for( let i = 0; i < data.data.length; i += 4 ){
data.data[i+3] = data.data[i];
}
// Put in the data
outputCtx.putImageData( data, 0, 0 );
// Draw the masked video into the mask (see the globalCompositeOperation above)
outputCtx.drawImage( video, 0, 0, w, h );
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
所以我的问题是：当两个视频都是 运行 时，是否有更有效的方法来屏蔽另一个视频 中的黑色像素？我目前并不担心并发性（两个流之间的差异小于一毫秒，远小于视频本身的实际帧）。 目标是提高帧率。
这是更广泛的代码片段，不幸的是，由于明显的 CORS 和视频托管原因，它无法处理堆栈溢出：
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const buffer = document.getElementById( 'buffer' );
const bufferCtx = buffer.getContext( '2d' );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );
const fpsOutput = document.getElementById( 'fps' );
const w = 1920;
const h = 1080;
let currentError;
let FPSCount = 0;
output.width =
buffer.width = w;
output.height =
buffer.height = h;
outputCtx.globalCompositeOperation = 'source-in';
function maskVideo(){
if( !video.paused ){
bufferCtx.drawImage( mask, 0, 0 );
const data = bufferCtx.getImageData( 0, 0, w, h );
for( let i = 0; i < data.data.length; i += 4 ){
data.data[i+3] = data.data[i];
}
outputCtx.putImageData( data, 0, 0 );
outputCtx.drawImage( video, 0, 0, w, h );
FPSCount++;
} else if( FPSCount ){
fpsOutput.textContent = (FPSCount / video.duration * video.playbackRate).toFixed(2);
FPSCount = 0;
}
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
html {
font-size: 1vh;
font-size: calc(var(--scale,1) * 1vh);
}
html, body {
height: 100%;
}
body {
padding: 0;
margin: 0;
overflow: hidden;
}
video {
display: none;
}
canvas {
position: fixed;
top: 0;
left: 0;
}
#fps {
position: fixed;
top: 10px;
left: 10px;
background: red;
color: white;
z-index: 5;
font-size: 50px;
margin: 0;
font-family: Courier, monospace;
font-variant-numeric: tabular-nums;
}
#fps:after {
content: 'fps';
}
<video id="video" src="./video.mp4" preload muted hidden></video>
<video id="mask" src="./mask.mp4" preload muted hidden></video>
<canvas id="output"></canvas>
<canvas id="buffer" hidden></canvas>
<p id="fps" hidden></p>
一个技巧是在较小版本的视频上做色键。
通常在视频上，您不会真正注意到掩蔽的质量低于实际可见视频。然而 CPU 会注意到它要比较的像素要少得多。
您甚至可以通过在蒙版上应用一个小的模糊滤镜来平滑边缘：
(async() => {
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const buffer = document.getElementById( 'buffer' );
// since we're going to do a lot of readbacks on the context
// we should let the browser know,
// so that it doesn't move it from and to the GPU every time
// For this we use the { willReadFrequently: true } option
const bufferCtx = buffer.getContext( '2d', { willReadFrequently: true } );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );
await video.play();
await mask.play();
const w = video.videoWidth;
const h = video.videoHeight;
output.width = w;
output.height = h;
buffer.width = mask.videoWidth / 5; // do the chroma on a smaller version
buffer.heigth = mask.videoHeight / 5;
function maskVideo(){
if( !video.paused ){
bufferCtx.drawImage( mask, 0, 0, buffer.width, buffer.height );
const data = bufferCtx.getImageData( 0, 0, buffer.width, buffer.height );
for( let i = 0; i < data.data.length; i += 4 ) {
data.data[i+3] = data.data[i];
}
// we put the new ImageData back on buffer
// so that we can stretch it on the output context
// alternatively we could have created an ImageBitmap from the ImageData
bufferCtx.putImageData( data, 0, 0 );
outputCtx.clearRect(0, 0, w, h);
outputCtx.filter = "blur(1px)"; // smoothen the mask
outputCtx.drawImage( buffer, 0, 0, w, h );
outputCtx.globalCompositeOperation = 'source-in';
outputCtx.filter = "none";
outputCtx.drawImage( video, 0, 0, w, h );
outputCtx.globalCompositeOperation = 'source-over';
}
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
})();
<video id="video" src="https://dl8.webmfiles.org/big-buck-bunny_trailer.webm" preload muted hidden loop></video>
<!-- not a great mask video example since it's really in shades of grays -->
<video id="mask" preload muted loop hidden src="https://upload.wikimedia.org/wikipedia/commons/6/64/Plan_9_from_Outer_Space_%281959%29.webm" crossorigin="anonymous"></video>
<canvas id="output"></canvas>
<canvas id="buffer" hidden></canvas>
另一个解决方案，因为你的蒙版是白色和黑色的，所以使用 SVG 滤镜，特别是 <feColorMatrix type=luminanceToAlpha>。这应该将所有工作留在 GPU 端，并消除对中间缓冲区的需要 canvas，但请注意，Safari 浏览器仍然不支持该选项...
(async() => {
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );
await video.play();
await mask.play();
const w = output.width = video.videoWidth;
const h = output.height = video.videoHeight;
function maskVideo(){
outputCtx.clearRect(0, 0, w, h);
outputCtx.filter = "url(#lumToAlpha)";
outputCtx.drawImage( mask, 0, 0, w, h );
outputCtx.filter = "none";
outputCtx.globalCompositeOperation = 'source-in';
outputCtx.drawImage( video, 0, 0, w, h );
outputCtx.globalCompositeOperation = 'source-over';
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
})();
<video id="video" src="https://dl8.webmfiles.org/big-buck-bunny_trailer.webm" preload muted hidden loop></video>
<video id="mask" preload muted loop hidden src="https://upload.wikimedia.org/wikipedia/commons/6/64/Plan_9_from_Outer_Space_%281959%29.webm" crossorigin="anonymous"></video>
<canvas id="output"></canvas>
<svg width=0 height=0 style=visibility:hidden;position:absolute>
<filter id=lumToAlpha>
<feColorMatrix type="luminanceToAlpha" />
</filter>
</svg>
我一直在尝试创建一个从一个视频到另一个视频的动态遮罩。这两个视频都没有 alpha 通道（即便如此，现在浏览器对它的支持也很低），所以我试图用 canvas 解决这个问题。我设法在非常有限的周期内完成了它，但它仍然无法在我的输出中以合适的帧速率完成此操作（它达到大约 20 fps，并且至少应该达到 25/30 才能显得平滑）。它实际上是 运行 的单数形式，周围有其他元素，所以我需要提高它的效率。
由于保密原因，我不能真正分享实际的视频，但其中一个包含黑色和白色的 alpha 掩码 (mask.mp4)（以及介于两者之间的灰色阴影），而另一个视频内容可以是任何需要屏蔽的内容 (video.mp4).
我要改进的代码实际上是这样的（假设所有视频和canvas元素都是1920x1080）：
const video = document.getElementById( 'video' ); // `<video src="video.mp4"></video>
const mask = document.getElementById( 'mask' ); // `<video src="mask.mp4"></video>
const buffer = document.getElementById( 'buffer' ); // <canvas hidden></canvas>
const bufferCtx = buffer.getContext( '2d' );
const output = document.getElementById( 'output' ); // <canvas></canvas>
const outputCtx = output.getContext( '2d' );
outputCtx.globalCompositeOperation = 'source-in';
function maskVideo(){
// Draw the mask
bufferCtx.drawImage( mask, 0, 0 );
// Get image data
const data = bufferCtx.getImageData( 0, 0, w, h );
// Assign the grayscale value to the alpha in the image data
for( let i = 0; i < data.data.length; i += 4 ){
data.data[i+3] = data.data[i];
}
// Put in the data
outputCtx.putImageData( data, 0, 0 );
// Draw the masked video into the mask (see the globalCompositeOperation above)
outputCtx.drawImage( video, 0, 0, w, h );
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
所以我的问题是：当两个视频都是 运行 时，是否有更有效的方法来屏蔽另一个视频 中的黑色像素？我目前并不担心并发性（两个流之间的差异小于一毫秒，远小于视频本身的实际帧）。 目标是提高帧率。
这是更广泛的代码片段，不幸的是，由于明显的 CORS 和视频托管原因，它无法处理堆栈溢出：
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const buffer = document.getElementById( 'buffer' );
const bufferCtx = buffer.getContext( '2d' );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );
const fpsOutput = document.getElementById( 'fps' );
const w = 1920;
const h = 1080;
let currentError;
let FPSCount = 0;
output.width =
buffer.width = w;
output.height =
buffer.height = h;
outputCtx.globalCompositeOperation = 'source-in';
function maskVideo(){
if( !video.paused ){
bufferCtx.drawImage( mask, 0, 0 );
const data = bufferCtx.getImageData( 0, 0, w, h );
for( let i = 0; i < data.data.length; i += 4 ){
data.data[i+3] = data.data[i];
}
outputCtx.putImageData( data, 0, 0 );
outputCtx.drawImage( video, 0, 0, w, h );
FPSCount++;
} else if( FPSCount ){
fpsOutput.textContent = (FPSCount / video.duration * video.playbackRate).toFixed(2);
FPSCount = 0;
}
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
html {
font-size: 1vh;
font-size: calc(var(--scale,1) * 1vh);
}
html, body {
height: 100%;
}
body {
padding: 0;
margin: 0;
overflow: hidden;
}
video {
display: none;
}
canvas {
position: fixed;
top: 0;
left: 0;
}
#fps {
position: fixed;
top: 10px;
left: 10px;
background: red;
color: white;
z-index: 5;
font-size: 50px;
margin: 0;
font-family: Courier, monospace;
font-variant-numeric: tabular-nums;
}
#fps:after {
content: 'fps';
}
<video id="video" src="./video.mp4" preload muted hidden></video>
<video id="mask" src="./mask.mp4" preload muted hidden></video>
<canvas id="output"></canvas>
<canvas id="buffer" hidden></canvas>
<p id="fps" hidden></p>
一个技巧是在较小版本的视频上做色键。
通常在视频上，您不会真正注意到掩蔽的质量低于实际可见视频。然而 CPU 会注意到它要比较的像素要少得多。
您甚至可以通过在蒙版上应用一个小的模糊滤镜来平滑边缘：
(async() => {
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const buffer = document.getElementById( 'buffer' );
// since we're going to do a lot of readbacks on the context
// we should let the browser know,
// so that it doesn't move it from and to the GPU every time
// For this we use the { willReadFrequently: true } option
const bufferCtx = buffer.getContext( '2d', { willReadFrequently: true } );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );
await video.play();
await mask.play();
const w = video.videoWidth;
const h = video.videoHeight;
output.width = w;
output.height = h;
buffer.width = mask.videoWidth / 5; // do the chroma on a smaller version
buffer.heigth = mask.videoHeight / 5;
function maskVideo(){
if( !video.paused ){
bufferCtx.drawImage( mask, 0, 0, buffer.width, buffer.height );
const data = bufferCtx.getImageData( 0, 0, buffer.width, buffer.height );
for( let i = 0; i < data.data.length; i += 4 ) {
data.data[i+3] = data.data[i];
}
// we put the new ImageData back on buffer
// so that we can stretch it on the output context
// alternatively we could have created an ImageBitmap from the ImageData
bufferCtx.putImageData( data, 0, 0 );
outputCtx.clearRect(0, 0, w, h);
outputCtx.filter = "blur(1px)"; // smoothen the mask
outputCtx.drawImage( buffer, 0, 0, w, h );
outputCtx.globalCompositeOperation = 'source-in';
outputCtx.filter = "none";
outputCtx.drawImage( video, 0, 0, w, h );
outputCtx.globalCompositeOperation = 'source-over';
}
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
})();
<video id="video" src="https://dl8.webmfiles.org/big-buck-bunny_trailer.webm" preload muted hidden loop></video>
<!-- not a great mask video example since it's really in shades of grays -->
<video id="mask" preload muted loop hidden src="https://upload.wikimedia.org/wikipedia/commons/6/64/Plan_9_from_Outer_Space_%281959%29.webm" crossorigin="anonymous"></video>
<canvas id="output"></canvas>
<canvas id="buffer" hidden></canvas>
另一个解决方案，因为你的蒙版是白色和黑色的，所以使用 SVG 滤镜，特别是 <feColorMatrix type=luminanceToAlpha>。这应该将所有工作留在 GPU 端，并消除对中间缓冲区的需要 canvas，但请注意，Safari 浏览器仍然不支持该选项...
(async() => {
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );
await video.play();
await mask.play();
const w = output.width = video.videoWidth;
const h = output.height = video.videoHeight;
function maskVideo(){
outputCtx.clearRect(0, 0, w, h);
outputCtx.filter = "url(#lumToAlpha)";
outputCtx.drawImage( mask, 0, 0, w, h );
outputCtx.filter = "none";
outputCtx.globalCompositeOperation = 'source-in';
outputCtx.drawImage( video, 0, 0, w, h );
outputCtx.globalCompositeOperation = 'source-over';
window.requestAnimationFrame( maskVideo );
}
window.requestAnimationFrame( maskVideo );
})();
<video id="video" src="https://dl8.webmfiles.org/big-buck-bunny_trailer.webm" preload muted hidden loop></video>
<video id="mask" preload muted loop hidden src="https://upload.wikimedia.org/wikipedia/commons/6/64/Plan_9_from_Outer_Space_%281959%29.webm" crossorigin="anonymous"></video>
<canvas id="output"></canvas>
<svg width=0 height=0 style=visibility:hidden;position:absolute>
<filter id=lumToAlpha>
<feColorMatrix type="luminanceToAlpha" />
</filter>
</svg>