android原生Opengl ES 3.0 PBO渲染空屏
android native Opengl ES 3.0 PBO rendering empty screen
我正在使用 android 开发相机预览应用。
我的应用流程如下所示。
1) JAVA : 获取相机预览缓冲区并将其传递给 JNI
2) CPP : 使用 OpenGL ES 3.0
从相机预览缓冲区制作纹理
3) CPP : 使用 OpenGL ES 3.0
渲染纹理
因此，我的应用程序可以正常使用普通纹理 update/rendering 解决方法。没关系。
问题出在性能上。我必须使用 glTexSubImage2D(...) 将每一帧更新到 GPU 内存，这有点慢。
所以我正在尝试使用 PBO，但呈现的屏幕始终是空白的。
我不知道我错过了什么，有人可以帮忙吗？
这是我的代码：
#include "native-lib.h"
#include "textureloader.h"
#include "vecmath.h"
#include "glutil.h"
#include "logger.h"
GLuint program;
GLuint p_mvp, p_vertices, p_uvs, p_tex;
GLuint tex;
GLuint pboIds[2];
GLfloat vertices[] = {0,0,0,0,0,0,0,0};
GLfloat uvs[] = {0,1,1,1,1,0,0,0};
GLushort indices[] = {0,1,2,0,2,3};
int dataSize;
float zoom;
mat4 mvp;
bool usePBO = true;
bool doublePBO = false;
cv::Mat mat;
std::string vs =
"#version 300 es\n"
"in vec4 a_vertices;\n"
"in vec2 a_uvs;\n"
"uniform mat4 u_mvp;\n"
"out vec2 v_texCoord;\n"
"void main(){\n"
" gl_Position = u_mvp * a_vertices;\n"
" v_texCoord = a_uvs;\n"
"}\n";
std::string fs =
"#version 300 es\n"
"uniform sampler2D u_tex;\n"
"in vec2 v_texCoord;\n"
"out vec4 fragColor;\n"
"void main(){\n"
" fragColor = texture(u_tex, v_texCoord);\n"
" fragColor.a = 1.0f;\n"
"}\n";
JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_glInit(JNIEnv *env, jclass)
{
// create program
program = loadProgram(vs, fs, false);
// create texture basic data
p_vertices = glGetAttribLocation(program, "a_vertices");
p_uvs = glGetAttribLocation(program, "a_uvs");
p_mvp = glGetUniformLocation(program, "u_mvp");
p_tex = glGetUniformLocation(program, "u_tex");
// create PBO
if(usePBO) {
glGenBuffers(2, pboIds);
}
return 1;
}
JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_surfaceChanged(JNIEnv *env, jclass,
jint w, jint h)
{
glViewport(0, 0, w, h);
zoom = (w < h ? w : h) / 2;
vec4 target = vec4(w / 2, h / 2, 0, 0);
vec4 eye = target + vec4(0, 0, zoom, 1);
mat4 view(mat4::lookAt(eye, target, vec4(0, 1, 0, 0)));
mat4 proj(mat4::perspective(90, (float)w / (float)h, 1, 1000));
mvp = proj * view;
vertices[5] = vertices[7] = h;
vertices[0] = vertices[6] = w;
glGenTextures(1, texId);
glBindTexture(GL_TEXTURE_2D, *texId);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
if(usePBO) {
dataSize = w * h * 3;
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[0]);
glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[1]);
glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
}
return 1;
}
JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray,
jint w, jint h)
{
jbyte *buf = env->GetByteArrayElements(bytearray, 0);
if(mat.empty())
mat = cv::Mat(h+h/2, w, CV_8UC1, buf);
else
mat.data = reinterpret_cast<uchar*>(buf);
cv::cvtColor(mat, mat, CV_YUV2RGB_NV21);
rotateMat(mat, mat, -90);
if(usePBO) {
static int index = 0;
if(doublePBO) {
int nextIndex = 0;
index = (index + 1) % 2;
nextIndex = (index + 1) % 2;
// bind the texture and PBO
glBindTexture(GL_TEXTURE_2D, tex);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
// copy pixels from PBO to texture object
// Use offset instead of ponter.
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[nextIndex]);
glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, 0, GL_STREAM_DRAW);
GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT);
if (ptr) {
memcpy(ptr, mat.ptr(), dataSize);
glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer
}
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
}
else {
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]);
glBindTexture(GL_TEXTURE_2D, tex);
GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT);
if (ptr) {
// memcpy(ptr, mat.ptr(), dataSize);
memset(ptr, 255, dataSize);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer
}
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
}
}
else {
loadTextureWithMat(mat, &tex, GL_RGB, GL_RGB, GL_UNSIGNED_BYTE);
}
// enable depth test
glEnable(GL_DEPTH_TEST);
glDepthFunc(GL_LEQUAL);
// enable alpha blending option
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glClearColor(1.f, 0.f, 0.f, 0.f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glUseProgram(program);
glUniformMatrix4fv(p_mvp, 1, GL_FALSE, &mvp.x.x);
glEnableVertexAttribArray(p_vertices);
glVertexAttribPointer(p_vertices, 2, GL_FLOAT, GL_FALSE, 0, vertices);
glEnableVertexAttribArray(p_uvs);
glVertexAttribPointer(p_uvs, 2, GL_FLOAT, GL_FALSE, 0, uvs);
glUniform1i(p_tex, 0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, tex);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices);
glBindTexture(GL_TEXTURE_2D, 0);
glUseProgram(0);
// disable alpha blending option
glDisable(GL_BLEND);
// disable depth test
glDisable(GL_DEPTH_TEST);
env->ReleaseByteArrayElements(bytearray, buf, JNI_ABORT);
mat.release();
return 1;
}
void rotateMat(cv::Mat const &src, cv::Mat &dst, int angle)
{
if(angle == 270 || angle == -90){
// Rotate clockwise 270 degrees
cv::transpose(src, dst);
cv::flip(dst, dst, 0);
}else if(angle == 180 || angle == -180){
// Rotate clockwise 180 degrees
cv::flip(src, dst, -1);
}else if(angle == 90 || angle == -270){
// Rotate clockwise 90 degrees
cv::transpose(src, dst);
cv::flip(dst, dst, 1);
}else if(angle == 360 || angle == 0 || angle == -360){
if(src.data != dst.data){
src.copyTo(dst);
}
}
}
如果我不使用 PBO 一切正常（fps 除外）所以我假设检查 usePBO 和 doublePBO 标志内的代码就足够了。
经过一番挣扎，我终于发现了我的小错误（但很严重）。
我旋转了我的 cv::Mat，所以宽度和高度被调换了。
Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray,
jint w, jint h)
{
mat = cv::Mat(h+h/2, w, CV_8UC1, buf); // created w*(h*1.5) size mat
cv::cvtColor(mat, mat, CV_YUV2RGB_NV21); // This makes mat height from h*1.5 to h
rotateMat(mat, mat, -90); // Now our mat is h*w
}
并且我使用了宽度、高度和函数参数值（没有交换）。
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
我修改了上面的代码：
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mat.cols, mat.rows, GL_RGB, GL_UNSIGNED_BYTE, 0);
是的，一切正常:)
注意：
顺便说一句，性能看起来和以前几乎一样。
也许使用 PBO 并不总能保证性能提升...
我正在使用 android 开发相机预览应用。
我的应用流程如下所示。
1) JAVA : 获取相机预览缓冲区并将其传递给 JNI
2) CPP : 使用 OpenGL ES 3.0
从相机预览缓冲区制作纹理
3) CPP : 使用 OpenGL ES 3.0
因此，我的应用程序可以正常使用普通纹理 update/rendering 解决方法。没关系。
问题出在性能上。我必须使用 glTexSubImage2D(...) 将每一帧更新到 GPU 内存，这有点慢。
所以我正在尝试使用 PBO，但呈现的屏幕始终是空白的。
我不知道我错过了什么，有人可以帮忙吗？
这是我的代码：
#include "native-lib.h"
#include "textureloader.h"
#include "vecmath.h"
#include "glutil.h"
#include "logger.h"
GLuint program;
GLuint p_mvp, p_vertices, p_uvs, p_tex;
GLuint tex;
GLuint pboIds[2];
GLfloat vertices[] = {0,0,0,0,0,0,0,0};
GLfloat uvs[] = {0,1,1,1,1,0,0,0};
GLushort indices[] = {0,1,2,0,2,3};
int dataSize;
float zoom;
mat4 mvp;
bool usePBO = true;
bool doublePBO = false;
cv::Mat mat;
std::string vs =
"#version 300 es\n"
"in vec4 a_vertices;\n"
"in vec2 a_uvs;\n"
"uniform mat4 u_mvp;\n"
"out vec2 v_texCoord;\n"
"void main(){\n"
" gl_Position = u_mvp * a_vertices;\n"
" v_texCoord = a_uvs;\n"
"}\n";
std::string fs =
"#version 300 es\n"
"uniform sampler2D u_tex;\n"
"in vec2 v_texCoord;\n"
"out vec4 fragColor;\n"
"void main(){\n"
" fragColor = texture(u_tex, v_texCoord);\n"
" fragColor.a = 1.0f;\n"
"}\n";
JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_glInit(JNIEnv *env, jclass)
{
// create program
program = loadProgram(vs, fs, false);
// create texture basic data
p_vertices = glGetAttribLocation(program, "a_vertices");
p_uvs = glGetAttribLocation(program, "a_uvs");
p_mvp = glGetUniformLocation(program, "u_mvp");
p_tex = glGetUniformLocation(program, "u_tex");
// create PBO
if(usePBO) {
glGenBuffers(2, pboIds);
}
return 1;
}
JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_surfaceChanged(JNIEnv *env, jclass,
jint w, jint h)
{
glViewport(0, 0, w, h);
zoom = (w < h ? w : h) / 2;
vec4 target = vec4(w / 2, h / 2, 0, 0);
vec4 eye = target + vec4(0, 0, zoom, 1);
mat4 view(mat4::lookAt(eye, target, vec4(0, 1, 0, 0)));
mat4 proj(mat4::perspective(90, (float)w / (float)h, 1, 1000));
mvp = proj * view;
vertices[5] = vertices[7] = h;
vertices[0] = vertices[6] = w;
glGenTextures(1, texId);
glBindTexture(GL_TEXTURE_2D, *texId);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
if(usePBO) {
dataSize = w * h * 3;
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[0]);
glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[1]);
glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
}
return 1;
}
JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray,
jint w, jint h)
{
jbyte *buf = env->GetByteArrayElements(bytearray, 0);
if(mat.empty())
mat = cv::Mat(h+h/2, w, CV_8UC1, buf);
else
mat.data = reinterpret_cast<uchar*>(buf);
cv::cvtColor(mat, mat, CV_YUV2RGB_NV21);
rotateMat(mat, mat, -90);
if(usePBO) {
static int index = 0;
if(doublePBO) {
int nextIndex = 0;
index = (index + 1) % 2;
nextIndex = (index + 1) % 2;
// bind the texture and PBO
glBindTexture(GL_TEXTURE_2D, tex);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
// copy pixels from PBO to texture object
// Use offset instead of ponter.
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[nextIndex]);
glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, 0, GL_STREAM_DRAW);
GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT);
if (ptr) {
memcpy(ptr, mat.ptr(), dataSize);
glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer
}
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
}
else {
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]);
glBindTexture(GL_TEXTURE_2D, tex);
GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT);
if (ptr) {
// memcpy(ptr, mat.ptr(), dataSize);
memset(ptr, 255, dataSize);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer
}
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
}
}
else {
loadTextureWithMat(mat, &tex, GL_RGB, GL_RGB, GL_UNSIGNED_BYTE);
}
// enable depth test
glEnable(GL_DEPTH_TEST);
glDepthFunc(GL_LEQUAL);
// enable alpha blending option
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glClearColor(1.f, 0.f, 0.f, 0.f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glUseProgram(program);
glUniformMatrix4fv(p_mvp, 1, GL_FALSE, &mvp.x.x);
glEnableVertexAttribArray(p_vertices);
glVertexAttribPointer(p_vertices, 2, GL_FLOAT, GL_FALSE, 0, vertices);
glEnableVertexAttribArray(p_uvs);
glVertexAttribPointer(p_uvs, 2, GL_FLOAT, GL_FALSE, 0, uvs);
glUniform1i(p_tex, 0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, tex);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices);
glBindTexture(GL_TEXTURE_2D, 0);
glUseProgram(0);
// disable alpha blending option
glDisable(GL_BLEND);
// disable depth test
glDisable(GL_DEPTH_TEST);
env->ReleaseByteArrayElements(bytearray, buf, JNI_ABORT);
mat.release();
return 1;
}
void rotateMat(cv::Mat const &src, cv::Mat &dst, int angle)
{
if(angle == 270 || angle == -90){
// Rotate clockwise 270 degrees
cv::transpose(src, dst);
cv::flip(dst, dst, 0);
}else if(angle == 180 || angle == -180){
// Rotate clockwise 180 degrees
cv::flip(src, dst, -1);
}else if(angle == 90 || angle == -270){
// Rotate clockwise 90 degrees
cv::transpose(src, dst);
cv::flip(dst, dst, 1);
}else if(angle == 360 || angle == 0 || angle == -360){
if(src.data != dst.data){
src.copyTo(dst);
}
}
}
如果我不使用 PBO 一切正常（fps 除外）所以我假设检查 usePBO 和 doublePBO 标志内的代码就足够了。
经过一番挣扎，我终于发现了我的小错误（但很严重）。
我旋转了我的 cv::Mat，所以宽度和高度被调换了。
Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray,
jint w, jint h)
{
mat = cv::Mat(h+h/2, w, CV_8UC1, buf); // created w*(h*1.5) size mat
cv::cvtColor(mat, mat, CV_YUV2RGB_NV21); // This makes mat height from h*1.5 to h
rotateMat(mat, mat, -90); // Now our mat is h*w
}
并且我使用了宽度、高度和函数参数值（没有交换）。
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
我修改了上面的代码：
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mat.cols, mat.rows, GL_RGB, GL_UNSIGNED_BYTE, 0);
是的，一切正常:)
注意：
顺便说一句，性能看起来和以前几乎一样。
也许使用 PBO 并不总能保证性能提升...