将 Kinect for Windows v2 的深度和颜色流保存为图像文件（例如 png 或 jpg）
Save depth and color streams of Kinect for Windows v2 as image files(e.g. png or jpg)
我正在尝试编写一个应用程序，将 Windows v2 的 Kinect 深度和颜色流保存为图像文件（如 png 或 jpg）。因此，我使用了 Kinect SDK v2 示例（因为我之前没有使用 C# 或 Kinect API 的经验）。我修改了 ColorBasics-WPF 示例代码以实现我的目标。这是仅将颜色流转换为 png 文件的代码（我修改的唯一部分是 Reader_ColorFrameArrived 函数）：
//------------------------------------------------------------------------------
// <copyright file="MainWindow.xaml.cs" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//------------------------------------------------------------------------------
namespace Microsoft.Samples.Kinect.ColorBasics
{
using System;
using System.ComponentModel;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using Microsoft.Kinect;
using System.Collections.Generic;
/// <summary>
/// Interaction logic for MainWindow
/// </summary>
public partial class MainWindow : Window, INotifyPropertyChanged
{
static int count = 0;
/// <summary>
/// Active Kinect sensor
/// </summary>
private KinectSensor kinectSensor = null;
/// <summary>
/// Reader for color frames
/// </summary>
private ColorFrameReader colorFrameReader = null;
/// <summary>
/// Bitmap to display
/// </summary>
private WriteableBitmap colorBitmap = null;
/// <summary>
/// Current status text to display
/// </summary>
private string statusText = null;
/// <summary>
/// Initializes a new instance of the MainWindow class.
/// </summary>
public MainWindow()
{
// get the kinectSensor object
this.kinectSensor = KinectSensor.GetDefault();
// open the reader for the color frames
this.colorFrameReader = this.kinectSensor.ColorFrameSource.OpenReader();
// wire handler for frame arrival
this.colorFrameReader.FrameArrived += this.Reader_ColorFrameArrived;
// create the colorFrameDescription from the ColorFrameSource using Bgra format
FrameDescription colorFrameDescription = this.kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Bgra);
// create the bitmap to display
this.colorBitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height, 96.0, 96.0, PixelFormats.Bgr32, null);
// set IsAvailableChanged event notifier
this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;
// open the sensor
this.kinectSensor.Open();
// set the status text
this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
: Properties.Resources.NoSensorStatusText;
// use the window object as the view model in this simple example
this.DataContext = this;
// initialize the components (controls) of the window
this.InitializeComponent();
}
/// <summary>
/// INotifyPropertyChangedPropertyChanged event to allow window controls to bind to changeable data
/// </summary>
public event PropertyChangedEventHandler PropertyChanged;
/// <summary>
/// Gets the bitmap to display
/// </summary>
public ImageSource ImageSource
{
get
{
return this.colorBitmap;
}
}
/// <summary>
/// Gets or sets the current status text to display
/// </summary>
public string StatusText
{
get
{
return this.statusText;
}
set
{
if (this.statusText != value)
{
this.statusText = value;
// notify any bound elements that the text has changed
if (this.PropertyChanged != null)
{
this.PropertyChanged(this, new PropertyChangedEventArgs("StatusText"));
}
}
}
}
/// <summary>
/// Execute shutdown tasks
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void MainWindow_Closing(object sender, CancelEventArgs e)
{
if (this.colorFrameReader != null)
{
// ColorFrameReder is IDisposable
this.colorFrameReader.Dispose();
this.colorFrameReader = null;
}
if (this.kinectSensor != null)
{
this.kinectSensor.Close();
this.kinectSensor = null;
}
}
/// <summary>
/// Handles the user clicking on the screenshot button
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void ScreenshotButton_Click(object sender, RoutedEventArgs e)
{
if (this.colorBitmap != null)
{
// create a png bitmap encoder which knows how to save a .png file
BitmapEncoder encoder = new PngBitmapEncoder();
// create frame from the writable bitmap and add to encoder
encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));
//bitmaps.Add(BitmapFrame.Create(this.colorBitmap.Clone()));
string time = count.ToString();
//string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);
string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");
count++;
// write the new file to disk
try
{
// FileStream is IDisposable
using (FileStream fs = new FileStream(path, FileMode.Create))
{
encoder.Save(fs);
}
this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
}
catch (IOException)
{
this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
}
}
}
/// <summary>
/// Handles the color frame data arriving from the sensor
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void Reader_ColorFrameArrived(object sender, ColorFrameArrivedEventArgs e)
{
// ColorFrame is IDisposable
using (ColorFrame colorFrame = e.FrameReference.AcquireFrame())
{
if (colorFrame != null)
{
FrameDescription colorFrameDescription = colorFrame.FrameDescription;
using (KinectBuffer colorBuffer = colorFrame.LockRawImageBuffer())
{
this.colorBitmap.Lock();
// verify data and write the new color frame data to the display bitmap
if ((colorFrameDescription.Width == this.colorBitmap.PixelWidth) && (colorFrameDescription.Height == this.colorBitmap.PixelHeight))
{
colorFrame.CopyConvertedFrameDataToIntPtr(
this.colorBitmap.BackBuffer,
(uint)(colorFrameDescription.Width * colorFrameDescription.Height * 4),
ColorImageFormat.Bgra);
this.colorBitmap.AddDirtyRect(new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight));
}
this.colorBitmap.Unlock();
}
}
// my modification : save current frame as png file.
if (this.colorBitmap != null)
{
// create a png bitmap encoder which knows how to save a .png file
BitmapEncoder encoder = new PngBitmapEncoder();
// create frame from the writable bitmap and add to encoder
encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));
string time = count.ToString();
//string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);
string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");
count++;
// write the new file to disk
try
{
// FileStream is IDisposable
using (FileStream fs = new FileStream(path, FileMode.Create))
{
encoder.Save(fs);
}
this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
}
catch (IOException)
{
this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
}
}
}
}
/// <summary>
/// Handles the event which the sensor becomes unavailable (E.g. paused, closed, unplugged).
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void Sensor_IsAvailableChanged(object sender, IsAvailableChangedEventArgs e)
{
// on failure, set the status text
this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
: Properties.Resources.SensorNotAvailableStatusText;
}
}
}
这段代码的问题是它在录制期间生成了 200 个 png 文件，但是从第 90 帧到最后，所有帧都彼此相同（它在我 运行 它并在我关闭它时停止。
1) 你能帮我理解为什么会这样吗？为什么它不记录剩余的帧并一次又一次地重复一个帧？
2) 关于我如何使用 Kinect for [=21] 以有效的方式并以良好的帧速率（例如 20-30 fps）同时将深度和颜色流记录为图像文件，您有什么建议或指示吗？ =] v2?
在不丢失帧的情况下以相同的 fps 同时写入颜色和深度流有点棘手。为了检查相同帧的问题，我建议您尝试将颜色和深度图像写入单独的缓冲区（您也可以将其帧时间戳保存在缓冲区中）并在录制结束后将它们写入磁盘。我会给你一个例子，你可以根据你的问题进行调整。
将彩色帧保存到缓冲区：
private void myKinectSensor_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
{
using (ColorImageFrame color = e.OpenColorImageFrame())
{
if (color != null)
{
colorbits = new byte[color.PixelDataLength];
color.CopyPixelDataTo(colorbits);
image1.Source = BitmapSource.Create(color.Width, color.Height, 96, 96, PixelFormats.Bgr32, null, colorbits, color.Width * color.BytesPerPixel);
if (StartSavingFrames)
{
SaveColorTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
SaveColorFrames.AddLast(colorbits);
}
}
}
}
将深度帧保存到另一个缓冲区：
private void myKinectSensor_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
{
using (DepthImageFrame depth = e.OpenDepthImageFrame())
{
depthPixels = new DepthImagePixel[myKinectSensor.DepthStream.FramePixelDataLength];
if (depth != null)
{
frame = new short[depth.PixelDataLength];
depth.CopyPixelDataTo(frame);
for (int i = 0; i < frame.Length; i++)
{
frame[i] = (short)(((ushort)frame[i]) >> 3);
}
image3.Source = BitmapSource.Create(depth.Width, depth.Height, 96, 96, PixelFormats.Gray16, null, frame, depth.Width * depth.BytesPerPixel);
if (StartSavingFrames)
{
SaveDepthTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
SaveDepthFrames.AddLast(frame);
}
}
}
}
最后，在每个缓冲区中循环并将帧写入磁盘：
e = SaveColorTimestamps.GetEnumerator();
foreach (byte[] node in SaveColorFrames)
{
e.MoveNext();
PngBitmapEncoder enc = new PngBitmapEncoder();
enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Bgr32, null, node, 640*4)));
string temppath = System.IO.Path.Combine(@"../output/kinect1/color/", e.Current + ".png");
using (FileStream fs = new FileStream(temppath, FileMode.Create))
{
enc.Save(fs);
fs.Close();
}
}
SaveColorTimestamps.Clear();
SaveColorFrames.Clear();
e.Dispose();
e = SaveDepthTimestamps.GetEnumerator();
foreach (short[] node in SaveDepthFrames)
{
e.MoveNext();
PngBitmapEncoder enc = new PngBitmapEncoder();
enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Gray16, null, node, 640 * 2)));
string temppath = System.IO.Path.Combine(@"../output/kinect1/depth/", e.Current + ".png");
using (FileStream fs = new FileStream(temppath, FileMode.Create))
{
enc.Save(fs);
fs.Close();
}
}
这不是最佳方法，但它会帮助您了解框架编写的工作原理，并且您不会有任何掉帧。
我正在尝试编写一个应用程序，将 Windows v2 的 Kinect 深度和颜色流保存为图像文件（如 png 或 jpg）。因此，我使用了 Kinect SDK v2 示例（因为我之前没有使用 C# 或 Kinect API 的经验）。我修改了 ColorBasics-WPF 示例代码以实现我的目标。这是仅将颜色流转换为 png 文件的代码（我修改的唯一部分是 Reader_ColorFrameArrived 函数）：
//------------------------------------------------------------------------------
// <copyright file="MainWindow.xaml.cs" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//------------------------------------------------------------------------------
namespace Microsoft.Samples.Kinect.ColorBasics
{
using System;
using System.ComponentModel;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using Microsoft.Kinect;
using System.Collections.Generic;
/// <summary>
/// Interaction logic for MainWindow
/// </summary>
public partial class MainWindow : Window, INotifyPropertyChanged
{
static int count = 0;
/// <summary>
/// Active Kinect sensor
/// </summary>
private KinectSensor kinectSensor = null;
/// <summary>
/// Reader for color frames
/// </summary>
private ColorFrameReader colorFrameReader = null;
/// <summary>
/// Bitmap to display
/// </summary>
private WriteableBitmap colorBitmap = null;
/// <summary>
/// Current status text to display
/// </summary>
private string statusText = null;
/// <summary>
/// Initializes a new instance of the MainWindow class.
/// </summary>
public MainWindow()
{
// get the kinectSensor object
this.kinectSensor = KinectSensor.GetDefault();
// open the reader for the color frames
this.colorFrameReader = this.kinectSensor.ColorFrameSource.OpenReader();
// wire handler for frame arrival
this.colorFrameReader.FrameArrived += this.Reader_ColorFrameArrived;
// create the colorFrameDescription from the ColorFrameSource using Bgra format
FrameDescription colorFrameDescription = this.kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Bgra);
// create the bitmap to display
this.colorBitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height, 96.0, 96.0, PixelFormats.Bgr32, null);
// set IsAvailableChanged event notifier
this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;
// open the sensor
this.kinectSensor.Open();
// set the status text
this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
: Properties.Resources.NoSensorStatusText;
// use the window object as the view model in this simple example
this.DataContext = this;
// initialize the components (controls) of the window
this.InitializeComponent();
}
/// <summary>
/// INotifyPropertyChangedPropertyChanged event to allow window controls to bind to changeable data
/// </summary>
public event PropertyChangedEventHandler PropertyChanged;
/// <summary>
/// Gets the bitmap to display
/// </summary>
public ImageSource ImageSource
{
get
{
return this.colorBitmap;
}
}
/// <summary>
/// Gets or sets the current status text to display
/// </summary>
public string StatusText
{
get
{
return this.statusText;
}
set
{
if (this.statusText != value)
{
this.statusText = value;
// notify any bound elements that the text has changed
if (this.PropertyChanged != null)
{
this.PropertyChanged(this, new PropertyChangedEventArgs("StatusText"));
}
}
}
}
/// <summary>
/// Execute shutdown tasks
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void MainWindow_Closing(object sender, CancelEventArgs e)
{
if (this.colorFrameReader != null)
{
// ColorFrameReder is IDisposable
this.colorFrameReader.Dispose();
this.colorFrameReader = null;
}
if (this.kinectSensor != null)
{
this.kinectSensor.Close();
this.kinectSensor = null;
}
}
/// <summary>
/// Handles the user clicking on the screenshot button
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void ScreenshotButton_Click(object sender, RoutedEventArgs e)
{
if (this.colorBitmap != null)
{
// create a png bitmap encoder which knows how to save a .png file
BitmapEncoder encoder = new PngBitmapEncoder();
// create frame from the writable bitmap and add to encoder
encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));
//bitmaps.Add(BitmapFrame.Create(this.colorBitmap.Clone()));
string time = count.ToString();
//string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);
string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");
count++;
// write the new file to disk
try
{
// FileStream is IDisposable
using (FileStream fs = new FileStream(path, FileMode.Create))
{
encoder.Save(fs);
}
this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
}
catch (IOException)
{
this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
}
}
}
/// <summary>
/// Handles the color frame data arriving from the sensor
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void Reader_ColorFrameArrived(object sender, ColorFrameArrivedEventArgs e)
{
// ColorFrame is IDisposable
using (ColorFrame colorFrame = e.FrameReference.AcquireFrame())
{
if (colorFrame != null)
{
FrameDescription colorFrameDescription = colorFrame.FrameDescription;
using (KinectBuffer colorBuffer = colorFrame.LockRawImageBuffer())
{
this.colorBitmap.Lock();
// verify data and write the new color frame data to the display bitmap
if ((colorFrameDescription.Width == this.colorBitmap.PixelWidth) && (colorFrameDescription.Height == this.colorBitmap.PixelHeight))
{
colorFrame.CopyConvertedFrameDataToIntPtr(
this.colorBitmap.BackBuffer,
(uint)(colorFrameDescription.Width * colorFrameDescription.Height * 4),
ColorImageFormat.Bgra);
this.colorBitmap.AddDirtyRect(new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight));
}
this.colorBitmap.Unlock();
}
}
// my modification : save current frame as png file.
if (this.colorBitmap != null)
{
// create a png bitmap encoder which knows how to save a .png file
BitmapEncoder encoder = new PngBitmapEncoder();
// create frame from the writable bitmap and add to encoder
encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));
string time = count.ToString();
//string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);
string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");
count++;
// write the new file to disk
try
{
// FileStream is IDisposable
using (FileStream fs = new FileStream(path, FileMode.Create))
{
encoder.Save(fs);
}
this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
}
catch (IOException)
{
this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
}
}
}
}
/// <summary>
/// Handles the event which the sensor becomes unavailable (E.g. paused, closed, unplugged).
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void Sensor_IsAvailableChanged(object sender, IsAvailableChangedEventArgs e)
{
// on failure, set the status text
this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
: Properties.Resources.SensorNotAvailableStatusText;
}
}
}
这段代码的问题是它在录制期间生成了 200 个 png 文件，但是从第 90 帧到最后，所有帧都彼此相同（它在我 运行 它并在我关闭它时停止。
1) 你能帮我理解为什么会这样吗？为什么它不记录剩余的帧并一次又一次地重复一个帧？
2) 关于我如何使用 Kinect for [=21] 以有效的方式并以良好的帧速率（例如 20-30 fps）同时将深度和颜色流记录为图像文件，您有什么建议或指示吗？ =] v2?
在不丢失帧的情况下以相同的 fps 同时写入颜色和深度流有点棘手。为了检查相同帧的问题，我建议您尝试将颜色和深度图像写入单独的缓冲区（您也可以将其帧时间戳保存在缓冲区中）并在录制结束后将它们写入磁盘。我会给你一个例子，你可以根据你的问题进行调整。
将彩色帧保存到缓冲区：
private void myKinectSensor_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
{
using (ColorImageFrame color = e.OpenColorImageFrame())
{
if (color != null)
{
colorbits = new byte[color.PixelDataLength];
color.CopyPixelDataTo(colorbits);
image1.Source = BitmapSource.Create(color.Width, color.Height, 96, 96, PixelFormats.Bgr32, null, colorbits, color.Width * color.BytesPerPixel);
if (StartSavingFrames)
{
SaveColorTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
SaveColorFrames.AddLast(colorbits);
}
}
}
}
将深度帧保存到另一个缓冲区：
private void myKinectSensor_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
{
using (DepthImageFrame depth = e.OpenDepthImageFrame())
{
depthPixels = new DepthImagePixel[myKinectSensor.DepthStream.FramePixelDataLength];
if (depth != null)
{
frame = new short[depth.PixelDataLength];
depth.CopyPixelDataTo(frame);
for (int i = 0; i < frame.Length; i++)
{
frame[i] = (short)(((ushort)frame[i]) >> 3);
}
image3.Source = BitmapSource.Create(depth.Width, depth.Height, 96, 96, PixelFormats.Gray16, null, frame, depth.Width * depth.BytesPerPixel);
if (StartSavingFrames)
{
SaveDepthTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
SaveDepthFrames.AddLast(frame);
}
}
}
}
最后，在每个缓冲区中循环并将帧写入磁盘：
e = SaveColorTimestamps.GetEnumerator();
foreach (byte[] node in SaveColorFrames)
{
e.MoveNext();
PngBitmapEncoder enc = new PngBitmapEncoder();
enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Bgr32, null, node, 640*4)));
string temppath = System.IO.Path.Combine(@"../output/kinect1/color/", e.Current + ".png");
using (FileStream fs = new FileStream(temppath, FileMode.Create))
{
enc.Save(fs);
fs.Close();
}
}
SaveColorTimestamps.Clear();
SaveColorFrames.Clear();
e.Dispose();
e = SaveDepthTimestamps.GetEnumerator();
foreach (short[] node in SaveDepthFrames)
{
e.MoveNext();
PngBitmapEncoder enc = new PngBitmapEncoder();
enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Gray16, null, node, 640 * 2)));
string temppath = System.IO.Path.Combine(@"../output/kinect1/depth/", e.Current + ".png");
using (FileStream fs = new FileStream(temppath, FileMode.Create))
{
enc.Save(fs);
fs.Close();
}
}
这不是最佳方法，但它会帮助您了解框架编写的工作原理，并且您不会有任何掉帧。