来自 YouTube 视频帧的 WebGL 纹理
WebGL textures from YouTube video frames
我正在使用 here (code, demo) for using video frames as WebGL textures, and the simple scene (just showing the image in 2D, rather than a 3D rotating cube) from here 中描述的技术。
目标是 YouTube 的 Tampermonkey 用户脚本（带有 WebGL 着色器，即视频效果）。
由于gl.clearColor(0.5,0.5,0.5,1)，canvas被填充为灰色。但是接下来应该从视频中绘制帧的代码行没有可见的效果。哪一部分可能是错误的？没有错误。
我试图在发布前缩短代码，但显然即使是简单的 WebGL 场景也需要大量样板代码。
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
uniform vec2 u_resolution;
varying vec2 v_texCoord;
void main() {
// convert the rectangle from pixels to 0.0 to 1.0
vec2 zeroToOne = a_position / u_resolution;
// convert from 0->1 to 0->2
vec2 zeroToTwo = zeroToOne * 2.0;
// convert from 0->2 to -1->+1 (clipspace)
vec2 clipSpace = zeroToTwo - 1.0;
gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
// pass the texCoord to the fragment shader
// The GPU will interpolate this value between points.
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
// our texture
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord).bgra;
}
`;
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource);
const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource);
// Create the shader program
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
// Send the source to the shader object
gl.shaderSource(shader, source);
// Compile the shader program
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize a shader program; this is where all the lighting
// for the vertices and so forth is established.
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
// Create a buffer to put three 2d clip space points in
var positionBuffer = gl.createBuffer();
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Set a rectangle the same size as the image.
setRectangle(gl, 0, 0, video.width, video.height);
// provide texture coordinates for the rectangle.
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]), gl.STATIC_DRAW);
// Create a texture.
var texture = initTexture(gl);
function drawScene() {
// lookup uniforms
var resolutionLocation = gl.getUniformLocation(shaderProgram, "u_resolution");
//webglUtils.resizeCanvasToDisplaySize(gl.canvas);
// Tell WebGL how to convert from clip space to pixels
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
// Clear the canvas
gl.clearColor(0.5,0.5,0.5,1);
gl.clear(gl.COLOR_BUFFER_BIT);
// Tell it to use our program (pair of shaders)
gl.useProgram(shaderProgram);
// Turn on the position attribute
gl.enableVertexAttribArray(positionLocation);
// Bind the position buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER)
var size = 2; // 2 components per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
positionLocation, size, type, normalize, stride, offset);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
// bind the texcoord buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
// Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER)
var size = 2; // 2 components per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
texcoordLocation, size, type, normalize, stride, offset);
// set the resolution
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
// Draw the rectangle.
var primitiveType = gl.TRIANGLES;
var offset = 0;
var count = 6;
gl.drawArrays(primitiveType, offset, count);
}
function setRectangle(gl, x, y, width, height) {
var x1 = x;
var x2 = x + width;
var y1 = y;
var y2 = y + height;
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
x1, y1,
x2, y1,
x1, y2,
x1, y2,
x2, y1,
x2, y2,
]), gl.STATIC_DRAW);
}
var then = 0;
// Draw the scene repeatedly
function render(now) {
now *= 0.001; // convert to seconds
const deltaTime = now - then;
then = now;
if (copyVideo) {
updateTexture(gl, texture, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
function initTexture(gl) {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
// Because video has to be download over the internet
// they might take a moment until it's ready so
// put a single pixel in the texture so we can
// use it immediately.
const level = 0;
const internalFormat = gl.RGBA;
const width = 1;
const height = 1;
const border = 0;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
width, height, border, srcFormat, srcType,
pixel);
// Turn off mips and set wrapping to clamp to edge so it
// will work regardless of the dimensions of the video.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
return texture;
}
function updateTexture(gl, texture, video) {
const level = 0;
const internalFormat = gl.RGBA;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
srcFormat, srcType, video);
}
})();
编辑：如前所述，此答案的前两部分完全错误。
TLDR：如果没有后端服务器首先获取视频数据，这可能不可行。
如果你查看你关注的MDN tutorial，传递给texImage2D的视频对象实际上是一个MP4视频。但是，在您的脚本中，您有权访问的视频对象 (document.getElementsByTagName("video")[0]) 只是一个 DOM 对象。您没有实际的视频数据。要访问 YouTube 并不容易。 YouTube 播放器不会一次性获取视频数据，而是 YouTube 流媒体服务器确保流式传输视频块。我对此不太确定，但我认为如果您的目标是获得实时视频效果，则很难解决这个问题。
我发现了一些关于此的讨论 (link1, link2)，这可能会有所帮助。
也就是说，从 WebGL 的角度来看，您的代码中存在一些问题。理想情况下，您拥有的代码应该显示一个蓝色矩形，因为这是您正在创建的纹理数据，而不是初始的 glClearColor 颜色。在视频开始播放后，它应该切换到视频纹理（由于我上面解释的问题，它会显示为黑色）。
我认为这是由于您在着色器中设置位置数据和进行剪辑 space 计算的方式所致。可以跳过直接发送归一化的设备坐标位置数据。这是更新后的代码，进行了一些清理以使其更短，其行为符合预期：
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
const texcoordData = new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture coordinate buffer
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, texcoordData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255, 255]));
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.vertexAttribPointer(texcoordLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo) {
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();
第一个问题已被 Atekihcan 正确指出，您将 NDC 坐标计算混淆了，实际上直接发送它们要容易得多。此外，您可以很容易地从中导出纹理坐标，从而保存第二个缓冲区的设置。
第二个问题是您的事件没有在您期望的链中触发（至少对我来说不是重新加载正在播放的视频和 运行 脚本）。我相信听 timeupdate 事件应该足够了，因为如果视频无法播放，时间将不会更新。工作代码：
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
video.addEventListener('timeupdate', function() {
copyVideo=true;
}, true);
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_position*.5+.5;
v_texCoord.y = 1.-v_texCoord.y;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, 1, 1, 0, gl.RGB, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255]));
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, video);
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();
注意：我还将纹理格式更改为 RGB（alpha 通道将隐式为 1），但这无关紧要。
我正在使用 here (code, demo) for using video frames as WebGL textures, and the simple scene (just showing the image in 2D, rather than a 3D rotating cube) from here 中描述的技术。
目标是 YouTube 的 Tampermonkey 用户脚本（带有 WebGL 着色器，即视频效果）。
由于gl.clearColor(0.5,0.5,0.5,1)，canvas被填充为灰色。但是接下来应该从视频中绘制帧的代码行没有可见的效果。哪一部分可能是错误的？没有错误。
我试图在发布前缩短代码，但显然即使是简单的 WebGL 场景也需要大量样板代码。
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
uniform vec2 u_resolution;
varying vec2 v_texCoord;
void main() {
// convert the rectangle from pixels to 0.0 to 1.0
vec2 zeroToOne = a_position / u_resolution;
// convert from 0->1 to 0->2
vec2 zeroToTwo = zeroToOne * 2.0;
// convert from 0->2 to -1->+1 (clipspace)
vec2 clipSpace = zeroToTwo - 1.0;
gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
// pass the texCoord to the fragment shader
// The GPU will interpolate this value between points.
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
// our texture
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord).bgra;
}
`;
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource);
const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource);
// Create the shader program
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
// Send the source to the shader object
gl.shaderSource(shader, source);
// Compile the shader program
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize a shader program; this is where all the lighting
// for the vertices and so forth is established.
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
// Create a buffer to put three 2d clip space points in
var positionBuffer = gl.createBuffer();
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Set a rectangle the same size as the image.
setRectangle(gl, 0, 0, video.width, video.height);
// provide texture coordinates for the rectangle.
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]), gl.STATIC_DRAW);
// Create a texture.
var texture = initTexture(gl);
function drawScene() {
// lookup uniforms
var resolutionLocation = gl.getUniformLocation(shaderProgram, "u_resolution");
//webglUtils.resizeCanvasToDisplaySize(gl.canvas);
// Tell WebGL how to convert from clip space to pixels
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
// Clear the canvas
gl.clearColor(0.5,0.5,0.5,1);
gl.clear(gl.COLOR_BUFFER_BIT);
// Tell it to use our program (pair of shaders)
gl.useProgram(shaderProgram);
// Turn on the position attribute
gl.enableVertexAttribArray(positionLocation);
// Bind the position buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER)
var size = 2; // 2 components per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
positionLocation, size, type, normalize, stride, offset);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
// bind the texcoord buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
// Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER)
var size = 2; // 2 components per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
texcoordLocation, size, type, normalize, stride, offset);
// set the resolution
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
// Draw the rectangle.
var primitiveType = gl.TRIANGLES;
var offset = 0;
var count = 6;
gl.drawArrays(primitiveType, offset, count);
}
function setRectangle(gl, x, y, width, height) {
var x1 = x;
var x2 = x + width;
var y1 = y;
var y2 = y + height;
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
x1, y1,
x2, y1,
x1, y2,
x1, y2,
x2, y1,
x2, y2,
]), gl.STATIC_DRAW);
}
var then = 0;
// Draw the scene repeatedly
function render(now) {
now *= 0.001; // convert to seconds
const deltaTime = now - then;
then = now;
if (copyVideo) {
updateTexture(gl, texture, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
function initTexture(gl) {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
// Because video has to be download over the internet
// they might take a moment until it's ready so
// put a single pixel in the texture so we can
// use it immediately.
const level = 0;
const internalFormat = gl.RGBA;
const width = 1;
const height = 1;
const border = 0;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
width, height, border, srcFormat, srcType,
pixel);
// Turn off mips and set wrapping to clamp to edge so it
// will work regardless of the dimensions of the video.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
return texture;
}
function updateTexture(gl, texture, video) {
const level = 0;
const internalFormat = gl.RGBA;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
srcFormat, srcType, video);
}
})();
编辑：如前所述，此答案的前两部分完全错误。
TLDR：如果没有后端服务器首先获取视频数据，这可能不可行。
如果你查看你关注的MDN tutorial，传递给texImage2D的视频对象实际上是一个MP4视频。但是，在您的脚本中，您有权访问的视频对象 (document.getElementsByTagName("video")[0]) 只是一个 DOM 对象。您没有实际的视频数据。要访问 YouTube 并不容易。 YouTube 播放器不会一次性获取视频数据，而是 YouTube 流媒体服务器确保流式传输视频块。我对此不太确定，但我认为如果您的目标是获得实时视频效果，则很难解决这个问题。
我发现了一些关于此的讨论 (link1, link2)，这可能会有所帮助。
也就是说，从 WebGL 的角度来看，您的代码中存在一些问题。理想情况下，您拥有的代码应该显示一个蓝色矩形，因为这是您正在创建的纹理数据，而不是初始的 glClearColor 颜色。在视频开始播放后，它应该切换到视频纹理（由于我上面解释的问题，它会显示为黑色）。
我认为这是由于您在着色器中设置位置数据和进行剪辑 space 计算的方式所致。可以跳过直接发送归一化的设备坐标位置数据。这是更新后的代码，进行了一些清理以使其更短，其行为符合预期：
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
const texcoordData = new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture coordinate buffer
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, texcoordData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255, 255]));
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.vertexAttribPointer(texcoordLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo) {
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();
第一个问题已被 Atekihcan 正确指出，您将 NDC 坐标计算混淆了，实际上直接发送它们要容易得多。此外，您可以很容易地从中导出纹理坐标，从而保存第二个缓冲区的设置。
第二个问题是您的事件没有在您期望的链中触发（至少对我来说不是重新加载正在播放的视频和 运行 脚本）。我相信听 timeupdate 事件应该足够了，因为如果视频无法播放，时间将不会更新。工作代码：
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
video.addEventListener('timeupdate', function() {
copyVideo=true;
}, true);
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_position*.5+.5;
v_texCoord.y = 1.-v_texCoord.y;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, 1, 1, 0, gl.RGB, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255]));
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, video);
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();
注意：我还将纹理格式更改为 RGB（alpha 通道将隐式为 1），但这无关紧要。