使用视频流作为开放的 GL ES 2.0 纹理
Using video stream as open GL ES 2.0 texture
我试图通过将 Open GL ES 纹理设置为 android surfaceTexture 来捕获视频并将其显示到屏幕上。我不能使用 TextureView 并按照 this tutorial 实施 SurfaceTextureListener，因为我使用的是 Google Cardboard。
我已经关注 the Android documentation on how to initialise Open GL ES 2.0 and use it, and also this tutorial 纹理。
将 2 放在一起我得到一个空白屏幕，偶尔会在控制台 window 中得到 <core_glBindTexture:572>: GL_INVALID_OPERATION。
被这么多我不知道的新概念弄得不知所措，我无法调试或者只是理解这两种方法是否可以这样使用。这是我的绘图代码，它在 MainActivity class 的 onSurfaceCreated() 中初始化，并从 Cardboard 的绘图函数 onEyeDraw() 中绘制。
package com.example.rich.test3;
import android.hardware.Camera;
import android.opengl.GLES20;
import android.view.TextureView;
import java.nio.ShortBuffer;
import java.nio.FloatBuffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Created by rich on 03/05/2015.
*/
public class Square {
private java.nio.FloatBuffer vertexBuffer;
private java.nio.ShortBuffer drawListBuffer;
private final java.nio.FloatBuffer mCubeTextureCoordinates;
float color[] = { 1.f, 1.f, 1.f, 1.0f };
private final String vertexShaderCode =
"attribute vec4 vPosition;" +
"attribute vec2 a_TexCoordinate;" +
"varying vec2 v_TexCoordinate;" +
"void main() {" +
" gl_Position = vPosition;" +
" v_TexCoordinate = a_TexCoordinate;" +
"}";
private final String fragmentShaderCode =
"precision mediump float;" +
"uniform vec4 vColor;" +
"uniform sampler2D u_Texture;" +
"varying vec2 v_TexCoordinate;" +
"void main() {" +
"gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
"}";
// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static float squareCoords[] = {
-0.5f, -0.5f, 0.0f, // bottom left
0.5f, -0.5f, 0.0f, // bottom right
-0.5f, 0.5f, 0.0f, // top left
0.5f, 0.5f, 0.0f}; // top right
private short drawOrder[] = { 0, 1, 2, 0, 2, 3 }; // order to draw vertices
private int mProgram;
private int mPositionHandle;
private int mColorHandle;
private int mTextureUniformHandle;
private int mTextureCoordinateHandle;
private final int mTextureCoordinateDataSize = 2;
private final int vertexCount = squareCoords.length / COORDS_PER_VERTEX;
private final int vertexStride = COORDS_PER_VERTEX * 4; // 4 bytes per vertex
private int mTextureDataHandle;
float textureCoordinates[] =
{0.0f, 1.0f,
1.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f };
Camera _camera;
TextureView _textureView;
int[] textures;
android.graphics.SurfaceTexture _surface;
public Square()
{
ByteBuffer bb = ByteBuffer.allocateDirect(
// (# of coordinate values * 4 bytes per float)
squareCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
vertexBuffer = bb.asFloatBuffer();
vertexBuffer.put(squareCoords);
vertexBuffer.position(0);
// initialize byte buffer for the draw list
ByteBuffer dlb = ByteBuffer.allocateDirect(
// (# of coordinate values * 2 bytes per short)
drawOrder.length * 2);
dlb.order(ByteOrder.nativeOrder());
drawListBuffer = dlb.asShortBuffer();
drawListBuffer.put(drawOrder);
drawListBuffer.position(0);
mCubeTextureCoordinates = ByteBuffer.allocateDirect(textureCoordinates.length * 4)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mCubeTextureCoordinates.put(textureCoordinates).position(0);
// create empty OpenGL ES Program
mProgram = GLES20.glCreateProgram();
textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
_surface = new android.graphics.SurfaceTexture(textures[0]);
_camera = Camera.open();
Camera.Size previewSize = _camera.getParameters().getPreviewSize();
try
{
_camera.setPreviewTexture(_surface);
}
catch (java.io.IOException ex)
{
// Console.writeLine (ex.Message);
}
final int vertexShaderHandle = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
GLES20.glShaderSource(vertexShaderHandle, vertexShaderCode);
GLES20.glCompileShader(vertexShaderHandle);
final int[] compileStatus = new int[1];
GLES20.glGetShaderiv(vertexShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
if (compileStatus[0] == 0)
{
//do check here
}
final int fragmentShaderHandle = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
GLES20.glShaderSource(fragmentShaderHandle, fragmentShaderCode);
GLES20.glCompileShader(fragmentShaderHandle);
GLES20.glGetShaderiv(fragmentShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
if (compileStatus[0] == 0)
{
//do check here
}
GLES20.glAttachShader(mProgram, vertexShaderHandle);
GLES20.glAttachShader(mProgram, fragmentShaderHandle);
GLES20.glBindAttribLocation(mProgram, 0, "a_Position");
GLES20.glBindAttribLocation(mProgram, 0, "a_TexCoordinate");
GLES20.glLinkProgram(mProgram);
final int[] linkStatus = new int[1];
GLES20.glGetProgramiv(mProgram, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] == 0)
{
//do check here
}
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[0]);
mTextureDataHandle = textures[0];
// Set filtering
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
}
public void draw()
{
_surface.updateTexImage();
GLES20.glUseProgram(mProgram);
mTextureUniformHandle = GLES20.glGetUniformLocation(mProgram, "u_Texture");
mPositionHandle = GLES20.glGetAttribLocation(mProgram, "a_Position");
mColorHandle = GLES20.glGetAttribLocation(mProgram, "a_Color");
mTextureCoordinateHandle = GLES20.glGetAttribLocation(mProgram, "a_TexCoordinate");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureDataHandle);
GLES20.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX,
GLES20.GL_FLOAT, false,
vertexStride, vertexBuffer);
GLES20.glVertexAttribPointer(mTextureCoordinateHandle, mTextureCoordinateDataSize, GLES20.GL_FLOAT, false,
0, mCubeTextureCoordinates);
GLES20.glEnableVertexAttribArray(mTextureCoordinateHandle);
GLES20.glEnableVertexAttribArray(mPositionHandle);
GLES20.glUniform1i(mTextureUniformHandle, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, vertexCount);
GLES20.glDisableVertexAttribArray(mPositionHandle);
}
}
渲染SurfaceTexture纹理对象时，需要使用GL_TEXTURE_EXTERNAL_OES纹理目标：
The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the GL_OES_EGL_image_external OpenGL ES extension. This limits how the texture may be used. Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture must declare its use of this extension using, for example, an "#extension GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using the samplerExternalOES GLSL sampler type.
因此您需要像这样更改片段着色器，添加 #extension 声明并将纹理统一声明为 samplerExternalOES:
private final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;" +
"uniform vec4 vColor;" +
"uniform samplerExternalOES u_Texture;" +
"varying vec2 v_TexCoordinate;" +
"void main() {" +
"gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
"}";
同样在您的 draw() 函数中，像这样绑定纹理：
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureDataHandle);
您不能使用普通纹理来渲染相机或视频预览，您必须使用 GL_TEXTURE_EXTERNAL_OES 扩展。我有同样的问题，我在 github 上找到了一个完整的工作解决方案。项目的名称是 android_instacam .
Here you will find source code to study. If you want to see it in action directly on your device just go on play store here.
我试图通过将 Open GL ES 纹理设置为 android surfaceTexture 来捕获视频并将其显示到屏幕上。我不能使用 TextureView 并按照 this tutorial 实施 SurfaceTextureListener，因为我使用的是 Google Cardboard。
我已经关注 the Android documentation on how to initialise Open GL ES 2.0 and use it, and also this tutorial 纹理。
将 2 放在一起我得到一个空白屏幕，偶尔会在控制台 window 中得到 <core_glBindTexture:572>: GL_INVALID_OPERATION。
被这么多我不知道的新概念弄得不知所措，我无法调试或者只是理解这两种方法是否可以这样使用。这是我的绘图代码，它在 MainActivity class 的 onSurfaceCreated() 中初始化，并从 Cardboard 的绘图函数 onEyeDraw() 中绘制。
package com.example.rich.test3;
import android.hardware.Camera;
import android.opengl.GLES20;
import android.view.TextureView;
import java.nio.ShortBuffer;
import java.nio.FloatBuffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Created by rich on 03/05/2015.
*/
public class Square {
private java.nio.FloatBuffer vertexBuffer;
private java.nio.ShortBuffer drawListBuffer;
private final java.nio.FloatBuffer mCubeTextureCoordinates;
float color[] = { 1.f, 1.f, 1.f, 1.0f };
private final String vertexShaderCode =
"attribute vec4 vPosition;" +
"attribute vec2 a_TexCoordinate;" +
"varying vec2 v_TexCoordinate;" +
"void main() {" +
" gl_Position = vPosition;" +
" v_TexCoordinate = a_TexCoordinate;" +
"}";
private final String fragmentShaderCode =
"precision mediump float;" +
"uniform vec4 vColor;" +
"uniform sampler2D u_Texture;" +
"varying vec2 v_TexCoordinate;" +
"void main() {" +
"gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
"}";
// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static float squareCoords[] = {
-0.5f, -0.5f, 0.0f, // bottom left
0.5f, -0.5f, 0.0f, // bottom right
-0.5f, 0.5f, 0.0f, // top left
0.5f, 0.5f, 0.0f}; // top right
private short drawOrder[] = { 0, 1, 2, 0, 2, 3 }; // order to draw vertices
private int mProgram;
private int mPositionHandle;
private int mColorHandle;
private int mTextureUniformHandle;
private int mTextureCoordinateHandle;
private final int mTextureCoordinateDataSize = 2;
private final int vertexCount = squareCoords.length / COORDS_PER_VERTEX;
private final int vertexStride = COORDS_PER_VERTEX * 4; // 4 bytes per vertex
private int mTextureDataHandle;
float textureCoordinates[] =
{0.0f, 1.0f,
1.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f };
Camera _camera;
TextureView _textureView;
int[] textures;
android.graphics.SurfaceTexture _surface;
public Square()
{
ByteBuffer bb = ByteBuffer.allocateDirect(
// (# of coordinate values * 4 bytes per float)
squareCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
vertexBuffer = bb.asFloatBuffer();
vertexBuffer.put(squareCoords);
vertexBuffer.position(0);
// initialize byte buffer for the draw list
ByteBuffer dlb = ByteBuffer.allocateDirect(
// (# of coordinate values * 2 bytes per short)
drawOrder.length * 2);
dlb.order(ByteOrder.nativeOrder());
drawListBuffer = dlb.asShortBuffer();
drawListBuffer.put(drawOrder);
drawListBuffer.position(0);
mCubeTextureCoordinates = ByteBuffer.allocateDirect(textureCoordinates.length * 4)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mCubeTextureCoordinates.put(textureCoordinates).position(0);
// create empty OpenGL ES Program
mProgram = GLES20.glCreateProgram();
textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
_surface = new android.graphics.SurfaceTexture(textures[0]);
_camera = Camera.open();
Camera.Size previewSize = _camera.getParameters().getPreviewSize();
try
{
_camera.setPreviewTexture(_surface);
}
catch (java.io.IOException ex)
{
// Console.writeLine (ex.Message);
}
final int vertexShaderHandle = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
GLES20.glShaderSource(vertexShaderHandle, vertexShaderCode);
GLES20.glCompileShader(vertexShaderHandle);
final int[] compileStatus = new int[1];
GLES20.glGetShaderiv(vertexShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
if (compileStatus[0] == 0)
{
//do check here
}
final int fragmentShaderHandle = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
GLES20.glShaderSource(fragmentShaderHandle, fragmentShaderCode);
GLES20.glCompileShader(fragmentShaderHandle);
GLES20.glGetShaderiv(fragmentShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
if (compileStatus[0] == 0)
{
//do check here
}
GLES20.glAttachShader(mProgram, vertexShaderHandle);
GLES20.glAttachShader(mProgram, fragmentShaderHandle);
GLES20.glBindAttribLocation(mProgram, 0, "a_Position");
GLES20.glBindAttribLocation(mProgram, 0, "a_TexCoordinate");
GLES20.glLinkProgram(mProgram);
final int[] linkStatus = new int[1];
GLES20.glGetProgramiv(mProgram, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] == 0)
{
//do check here
}
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[0]);
mTextureDataHandle = textures[0];
// Set filtering
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
}
public void draw()
{
_surface.updateTexImage();
GLES20.glUseProgram(mProgram);
mTextureUniformHandle = GLES20.glGetUniformLocation(mProgram, "u_Texture");
mPositionHandle = GLES20.glGetAttribLocation(mProgram, "a_Position");
mColorHandle = GLES20.glGetAttribLocation(mProgram, "a_Color");
mTextureCoordinateHandle = GLES20.glGetAttribLocation(mProgram, "a_TexCoordinate");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureDataHandle);
GLES20.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX,
GLES20.GL_FLOAT, false,
vertexStride, vertexBuffer);
GLES20.glVertexAttribPointer(mTextureCoordinateHandle, mTextureCoordinateDataSize, GLES20.GL_FLOAT, false,
0, mCubeTextureCoordinates);
GLES20.glEnableVertexAttribArray(mTextureCoordinateHandle);
GLES20.glEnableVertexAttribArray(mPositionHandle);
GLES20.glUniform1i(mTextureUniformHandle, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, vertexCount);
GLES20.glDisableVertexAttribArray(mPositionHandle);
}
}
渲染SurfaceTexture纹理对象时，需要使用GL_TEXTURE_EXTERNAL_OES纹理目标：
The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the GL_OES_EGL_image_external OpenGL ES extension. This limits how the texture may be used. Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture must declare its use of this extension using, for example, an "#extension GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using the samplerExternalOES GLSL sampler type.
因此您需要像这样更改片段着色器，添加 #extension 声明并将纹理统一声明为 samplerExternalOES:
private final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;" +
"uniform vec4 vColor;" +
"uniform samplerExternalOES u_Texture;" +
"varying vec2 v_TexCoordinate;" +
"void main() {" +
"gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
"}";
同样在您的 draw() 函数中，像这样绑定纹理：
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureDataHandle);
您不能使用普通纹理来渲染相机或视频预览，您必须使用 GL_TEXTURE_EXTERNAL_OES 扩展。我有同样的问题，我在 github 上找到了一个完整的工作解决方案。项目的名称是 android_instacam .
Here you will find source code to study. If you want to see it in action directly on your device just go on play store here.