从点云生成深度图
Generating depth map from point cloud
我正在尝试从点云生成深度图。我知道我可以将点云投影到图像平面，但是 TangoSupport 脚本中已经有一个函数 (ScreenCoordinateToWorldNearestNeighbor) 可以找到给定屏幕坐标的 XYZ 点。
我无法使用此支持功能，似乎我的一项或多项输入无效。我正在 OnTangoDepthAvailable 事件中更新我的深度图纹理。
public void OnTangoDepthAvailable(TangoUnityDepth tangoDepth)
{
_depthAvailable = true;
Matrix4x4 ccWorld = _Camera.transform.localToWorldMatrix;
bool isValid = false;
Vector3 colorCameraPoint = new Vector3();
for (int i = 0; i < _depthMapSize; i++)
{
for (int j = 0; j < _depthMapSize; j++)
{
if (TangoSupport.ScreenCoordinateToWorldNearestNeighbor(
_PointCloud.m_points, _PointCloud.m_pointsCount,
tangoDepth.m_timestamp,
_ccIntrinsics,
ref ccWorld,
new Vector2(i / (float)_depthMapSize, j / (float)_depthMapSize),
out colorCameraPoint, out isValid) == Common.ErrorType.TANGO_INVALID)
{
_depthTexture.SetPixel(i, j, Color.red);
continue;
}
if (isValid)
{
//_depthTexture.SetPixel(i, j, new Color(colorCameraPoint.z, colorCameraPoint.z, colorCameraPoint.z));
_depthTexture.SetPixel(i, j,
new Color(0,UnityEngine.Random.value,0));
}
else
{
_depthTexture.SetPixel(i, j, Color.white);
}
}
}
_depthTexture.Apply();
_DepthMapQuad.material.mainTexture = _depthTexture;
}
如果非要我猜的话，我会说我传入了错误的矩阵 (ccWorld)。这是它在矩阵参数的文档中所说的：
Transformation matrix of the color camera with respect to the Unity
world frame.
结果是一张白色的深度图，说明函数return成功，但是isValid为false，表示投影后找不到附近的点云点。
有什么想法吗？我还注意到性能很差，即使我的深度图是 8x8。当有新的深度数据可用时（在 OnTangoDepthAvailable 内），我不应该更新深度图吗？
编辑：
我能够成功地使函数 return 成功，但是现在它在投影后找不到附近的点云点。生成的深度图始终为白色。我正在打印所有参数，看起来都是正确的，所以我想我传入了错误的矩阵。
您应该更新您的 SDK 和 Project Tango 开发工具包。这是在 Android 上获取深度图 的示例，也许您会得到 unity:
的提示
public class MainActivity extends AppCompatActivity {
private Tango mTango;
private TangoConfig mTangoConfig;
private TangoPointCloudManager mPointCloudManager;
private AtomicBoolean tConnected = new AtomicBoolean(false);
Random rand = new Random();
private ImageView imageDepthMap;
private static final ArrayList<TangoCoordinateFramePair> framePairs = new ArrayList<TangoCoordinateFramePair>();
{
framePairs.add(new TangoCoordinateFramePair(
TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH,
TangoPoseData.COORDINATE_FRAME_DEVICE));
}
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
//initialize the imageView
imageDepthMap = (ImageView)findViewById(R.id.imageView);
//initialize pointCloudManager
mPointCloudManager = new TangoPointCloudManager();
}
@Override
protected void onResume(){
super.onResume();
//obtain the tango configuration
if(tConnected.compareAndSet(false, true)) {
try {
setTango();
} catch (TangoOutOfDateException tE) {
tE.printStackTrace();
}
}
}
@Override
protected void onPause(){
super.onPause();
if(tConnected.compareAndSet(true, false)) {
try {
//disconnect Tango service so other applications can use it
mTango.disconnect();
} catch (TangoException e) {
e.printStackTrace();
}
}
}
private void setTango(){
mTango = new Tango(MainActivity.this, new Runnable() {
@Override
public void run() {
TangoSupport.initialize();
mTangoConfig = new TangoConfig();
mTangoConfig = mTango.getConfig(TangoConfig.CONFIG_TYPE_CURRENT);
mTangoConfig.putBoolean(TangoConfig.KEY_BOOLEAN_DEPTH, true); //activate depth sensing
mTango.connect(mTangoConfig);
mTango.connectListener(framePairs, new Tango.OnTangoUpdateListener() {
@Override
public void onPoseAvailable(TangoPoseData tangoPoseData) {
}
@Override
public void onXyzIjAvailable(TangoXyzIjData pointCloud) {
// Log.d("gDebug", "xyZAvailable");
//TangoXyzIjData pointCloud = mPointCloudManager.getLatestXyzIj();
// Update current camera pose
if (pointCloud.ijRows * pointCloud.ijCols > 0){
try {
// Calculate the last camera color pose.
TangoPoseData lastFramePose = TangoSupport.getPoseAtTime(0,
TangoPoseData.COORDINATE_FRAME_START_OF_SERVICE,
TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR,
TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, 0);
if (pointCloud != null) {
//obtain depth info per pixel
TangoSupport.DepthBuffer depthBuf = TangoSupport.upsampleImageNearestNeighbor(pointCloud, mTango.getCameraIntrinsics(TangoCameraIntrinsics.TANGO_CAMERA_COLOR), lastFramePose);
//create Depth map
int[] intBuff = convertToInt(depthBuf.depths, depthBuf.width, depthBuf.height);
final Bitmap Image = Bitmap.createBitmap(intBuff, depthBuf.width, depthBuf.height, Bitmap.Config.ARGB_8888);
runOnUiThread(new Runnable() {
@Override
public void run() {
imageDepthMap.setImageBitmap(Image);
}
});
}
} catch (TangoErrorException e) {
Log.e("gDebug", "Could not get valid transform");
}
}
}
@Override
public void onFrameAvailable(int i) {
//Log.d("gDebug", "Frame Available from " + i);
}
@Override
public void onTangoEvent(TangoEvent tangoEvent) {
}
});
}
});
}
private int[] convertToInt(FloatBuffer pointCloudData, int width, int height){
double mulFact = 255.0/5.0;
int byteArrayCapacity = width * height;
int[] depthMap = new int[byteArrayCapacity];
int grayPixVal = 0;
pointCloudData.rewind();
for(int i =0; i < byteArrayCapacity; i++){
//obtain grayscale representation
grayPixVal = (int)(mulFact * (5.0- pointCloudData.get(i)));
depthMap[i] = Color.rgb(grayPixVal, grayPixVal, grayPixVal);
}
return depthMap;
}
}
我从我已经工作的版本中提取了这段代码。尝试修复任何与配置相关的错误。该代码假定深度估计的深度感应范围为 0.4m - 5m。将零映射到 255 允许未估计的区域（值为零）为白色。
我正在尝试从点云生成深度图。我知道我可以将点云投影到图像平面，但是 TangoSupport 脚本中已经有一个函数 (ScreenCoordinateToWorldNearestNeighbor) 可以找到给定屏幕坐标的 XYZ 点。
我无法使用此支持功能，似乎我的一项或多项输入无效。我正在 OnTangoDepthAvailable 事件中更新我的深度图纹理。
public void OnTangoDepthAvailable(TangoUnityDepth tangoDepth)
{
_depthAvailable = true;
Matrix4x4 ccWorld = _Camera.transform.localToWorldMatrix;
bool isValid = false;
Vector3 colorCameraPoint = new Vector3();
for (int i = 0; i < _depthMapSize; i++)
{
for (int j = 0; j < _depthMapSize; j++)
{
if (TangoSupport.ScreenCoordinateToWorldNearestNeighbor(
_PointCloud.m_points, _PointCloud.m_pointsCount,
tangoDepth.m_timestamp,
_ccIntrinsics,
ref ccWorld,
new Vector2(i / (float)_depthMapSize, j / (float)_depthMapSize),
out colorCameraPoint, out isValid) == Common.ErrorType.TANGO_INVALID)
{
_depthTexture.SetPixel(i, j, Color.red);
continue;
}
if (isValid)
{
//_depthTexture.SetPixel(i, j, new Color(colorCameraPoint.z, colorCameraPoint.z, colorCameraPoint.z));
_depthTexture.SetPixel(i, j,
new Color(0,UnityEngine.Random.value,0));
}
else
{
_depthTexture.SetPixel(i, j, Color.white);
}
}
}
_depthTexture.Apply();
_DepthMapQuad.material.mainTexture = _depthTexture;
}
如果非要我猜的话，我会说我传入了错误的矩阵 (ccWorld)。这是它在矩阵参数的文档中所说的：
Transformation matrix of the color camera with respect to the Unity world frame.
结果是一张白色的深度图，说明函数return成功，但是isValid为false，表示投影后找不到附近的点云点。
有什么想法吗？我还注意到性能很差，即使我的深度图是 8x8。当有新的深度数据可用时（在 OnTangoDepthAvailable 内），我不应该更新深度图吗？
编辑： 我能够成功地使函数 return 成功，但是现在它在投影后找不到附近的点云点。生成的深度图始终为白色。我正在打印所有参数，看起来都是正确的，所以我想我传入了错误的矩阵。
您应该更新您的 SDK 和 Project Tango 开发工具包。这是在 Android 上获取深度图 的示例，也许您会得到 unity:
的提示public class MainActivity extends AppCompatActivity {
private Tango mTango;
private TangoConfig mTangoConfig;
private TangoPointCloudManager mPointCloudManager;
private AtomicBoolean tConnected = new AtomicBoolean(false);
Random rand = new Random();
private ImageView imageDepthMap;
private static final ArrayList<TangoCoordinateFramePair> framePairs = new ArrayList<TangoCoordinateFramePair>();
{
framePairs.add(new TangoCoordinateFramePair(
TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH,
TangoPoseData.COORDINATE_FRAME_DEVICE));
}
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
//initialize the imageView
imageDepthMap = (ImageView)findViewById(R.id.imageView);
//initialize pointCloudManager
mPointCloudManager = new TangoPointCloudManager();
}
@Override
protected void onResume(){
super.onResume();
//obtain the tango configuration
if(tConnected.compareAndSet(false, true)) {
try {
setTango();
} catch (TangoOutOfDateException tE) {
tE.printStackTrace();
}
}
}
@Override
protected void onPause(){
super.onPause();
if(tConnected.compareAndSet(true, false)) {
try {
//disconnect Tango service so other applications can use it
mTango.disconnect();
} catch (TangoException e) {
e.printStackTrace();
}
}
}
private void setTango(){
mTango = new Tango(MainActivity.this, new Runnable() {
@Override
public void run() {
TangoSupport.initialize();
mTangoConfig = new TangoConfig();
mTangoConfig = mTango.getConfig(TangoConfig.CONFIG_TYPE_CURRENT);
mTangoConfig.putBoolean(TangoConfig.KEY_BOOLEAN_DEPTH, true); //activate depth sensing
mTango.connect(mTangoConfig);
mTango.connectListener(framePairs, new Tango.OnTangoUpdateListener() {
@Override
public void onPoseAvailable(TangoPoseData tangoPoseData) {
}
@Override
public void onXyzIjAvailable(TangoXyzIjData pointCloud) {
// Log.d("gDebug", "xyZAvailable");
//TangoXyzIjData pointCloud = mPointCloudManager.getLatestXyzIj();
// Update current camera pose
if (pointCloud.ijRows * pointCloud.ijCols > 0){
try {
// Calculate the last camera color pose.
TangoPoseData lastFramePose = TangoSupport.getPoseAtTime(0,
TangoPoseData.COORDINATE_FRAME_START_OF_SERVICE,
TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR,
TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, 0);
if (pointCloud != null) {
//obtain depth info per pixel
TangoSupport.DepthBuffer depthBuf = TangoSupport.upsampleImageNearestNeighbor(pointCloud, mTango.getCameraIntrinsics(TangoCameraIntrinsics.TANGO_CAMERA_COLOR), lastFramePose);
//create Depth map
int[] intBuff = convertToInt(depthBuf.depths, depthBuf.width, depthBuf.height);
final Bitmap Image = Bitmap.createBitmap(intBuff, depthBuf.width, depthBuf.height, Bitmap.Config.ARGB_8888);
runOnUiThread(new Runnable() {
@Override
public void run() {
imageDepthMap.setImageBitmap(Image);
}
});
}
} catch (TangoErrorException e) {
Log.e("gDebug", "Could not get valid transform");
}
}
}
@Override
public void onFrameAvailable(int i) {
//Log.d("gDebug", "Frame Available from " + i);
}
@Override
public void onTangoEvent(TangoEvent tangoEvent) {
}
});
}
});
}
private int[] convertToInt(FloatBuffer pointCloudData, int width, int height){
double mulFact = 255.0/5.0;
int byteArrayCapacity = width * height;
int[] depthMap = new int[byteArrayCapacity];
int grayPixVal = 0;
pointCloudData.rewind();
for(int i =0; i < byteArrayCapacity; i++){
//obtain grayscale representation
grayPixVal = (int)(mulFact * (5.0- pointCloudData.get(i)));
depthMap[i] = Color.rgb(grayPixVal, grayPixVal, grayPixVal);
}
return depthMap;
}
}
我从我已经工作的版本中提取了这段代码。尝试修复任何与配置相关的错误。该代码假定深度估计的深度感应范围为 0.4m - 5m。将零映射到 255 允许未估计的区域（值为零）为白色。