读取 MNIST 数据库
Reading MNIST Database
我目前正在探索神经网络和机器学习,并且我在 C# 中实现了一个基本的神经网络。现在我想用 MNIST 数据库测试我的反向传播训练算法。虽然我在正确读取文件时遇到了严重的问题。
剧透代码目前针对性能进行了非常糟糕的优化。我目前的目标是掌握主题并获得结构化视图,然后再开始使用更快的数据结构。
为了训练网络,我想为其提供自定义 TrainingSet 数据结构:
[Serializable]
public class TrainingSet
{
public Dictionary<List<double>, List<double>> data = new Dictionary<List<double>, List<double>>();
}
键将是我的输入数据(每个条目(图像)784 像素,代表 0 到 1 范围内的灰度值)。值将是我的输出数据(10 个条目代表 0-9 的数字,所有条目都为 0,除了 exspected 为 1)
现在我想按照这个约定读取MNIST数据库。我目前正在进行第二次尝试,这是受这篇博文的启发:https://jamesmccaffrey.wordpress.com/2013/11/23/reading-the-mnist-data-set-with-c/。可悲的是,它仍然产生与我第一次尝试以奇怪的模式散布像素相同的废话:
我目前的阅读算法:
public static TrainingSet GenerateTrainingSet(FileInfo imagesFile, FileInfo labelsFile)
{
MnistImageView imageView = new MnistImageView();
imageView.Show();
TrainingSet trainingSet = new TrainingSet();
List<List<double>> labels = new List<List<double>>();
List<List<double>> images = new List<List<double>>();
using (BinaryReader brLabels = new BinaryReader(new FileStream(labelsFile.FullName, FileMode.Open)))
{
using (BinaryReader brImages = new BinaryReader(new FileStream(imagesFile.FullName, FileMode.Open)))
{
int magic1 = brImages.ReadBigInt32(); //Reading as BigEndian
int numImages = brImages.ReadBigInt32();
int numRows = brImages.ReadBigInt32();
int numCols = brImages.ReadBigInt32();
int magic2 = brLabels.ReadBigInt32();
int numLabels = brLabels.ReadBigInt32();
byte[] pixels = new byte[numRows * numCols];
// each image
for (int imageCounter = 0; imageCounter < numImages; imageCounter++)
{
List<double> imageInput = new List<double>();
List<double> exspectedOutput = new List<double>();
for (int i = 0; i < 10; i++) //generate empty exspected output
exspectedOutput.Add(0);
//read image
for (int p = 0; p < pixels.Length; p++)
{
byte b = brImages.ReadByte();
pixels[p] = b;
imageInput.Add(b / 255.0f); //scale in 0 to 1 range
}
//read label
byte lbl = brLabels.ReadByte();
exspectedOutput[lbl] = 1; //modify exspected output
labels.Add(exspectedOutput);
images.Add(imageInput);
//Debug view showing parsed image.......................
Bitmap image = new Bitmap(numCols, numRows);
for (int y = 0; y < numRows; y++)
{
for (int x = 0; x < numCols; x++)
{
image.SetPixel(x, y, Color.FromArgb(255 - pixels[x * y], 255 - pixels[x * y], 255 - pixels[x * y])); //invert colors to have 0,0,0 be white as specified by mnist
}
}
imageView.SetImage(image);
imageView.Refresh();
//.......................................................
}
brImages.Close();
brLabels.Close();
}
}
for (int i = 0; i < images.Count; i++)
{
trainingSet.data.Add(images[i], labels[i]);
}
return trainingSet;
}
所有图像都会产生如上所示的图案。它从来都不是完全相同的图案,但似乎总是有像素 "pulled" 到右下角。
我就是这样做的:
public static class MnistReader
{
private const string TrainImages = "mnist/train-images.idx3-ubyte";
private const string TrainLabels = "mnist/train-labels.idx1-ubyte";
private const string TestImages = "mnist/t10k-images.idx3-ubyte";
private const string TestLabels = "mnist/t10k-labels.idx1-ubyte";
public static IEnumerable<Image> ReadTrainingData()
{
foreach (var item in Read(TrainImages, TrainLabels))
{
yield return item;
}
}
public static IEnumerable<Image> ReadTestData()
{
foreach (var item in Read(TestImages, TestLabels))
{
yield return item;
}
}
private static IEnumerable<Image> Read(string imagesPath, string labelsPath)
{
BinaryReader labels = new BinaryReader(new FileStream(labelsPath, FileMode.Open));
BinaryReader images = new BinaryReader(new FileStream(imagesPath, FileMode.Open));
int magicNumber = images.ReadBigInt32();
int numberOfImages = images.ReadBigInt32();
int width = images.ReadBigInt32();
int height = images.ReadBigInt32();
int magicLabel = labels.ReadBigInt32();
int numberOfLabels = labels.ReadBigInt32();
for (int i = 0; i < numberOfImages; i++)
{
var bytes = images.ReadBytes(width * height);
var arr = new byte[height, width];
arr.ForEach((j,k) => arr[j, k] = bytes[j * height + k]);
yield return new Image()
{
Data = arr,
Label = labels.ReadByte()
};
}
}
}
Image
class:
public class Image
{
public byte Label { get; set; }
public byte[,] Data { get; set; }
}
一些扩展方法:
public static class Extensions
{
public static int ReadBigInt32(this BinaryReader br)
{
var bytes = br.ReadBytes(sizeof(Int32));
if (BitConverter.IsLittleEndian) Array.Reverse(bytes);
return BitConverter.ToInt32(bytes, 0);
}
public static void ForEach<T>(this T[,] source, Action<int, int> action)
{
for (int w = 0; w < source.GetLength(0); w++)
{
for (int h = 0; h < source.GetLength(1); h++)
{
action(w, h);
}
}
}
}
用法:
foreach (var image in MnistReader.ReadTrainingData())
{
//use image here
}
或
foreach (var image in MnistReader.ReadTestData())
{
//use image here
}
为什么不使用 nuget 包:
- MNIST.IO只是一个数据读取器(免责声明:我的包裹)
- Accord.DataSets包含类下载和解析MNIST、News20、Iris等机器学习数据集。这个包是 Accord.NET 框架的一部分。
我目前正在探索神经网络和机器学习,并且我在 C# 中实现了一个基本的神经网络。现在我想用 MNIST 数据库测试我的反向传播训练算法。虽然我在正确读取文件时遇到了严重的问题。
剧透代码目前针对性能进行了非常糟糕的优化。我目前的目标是掌握主题并获得结构化视图,然后再开始使用更快的数据结构。
为了训练网络,我想为其提供自定义 TrainingSet 数据结构:
[Serializable]
public class TrainingSet
{
public Dictionary<List<double>, List<double>> data = new Dictionary<List<double>, List<double>>();
}
键将是我的输入数据(每个条目(图像)784 像素,代表 0 到 1 范围内的灰度值)。值将是我的输出数据(10 个条目代表 0-9 的数字,所有条目都为 0,除了 exspected 为 1)
现在我想按照这个约定读取MNIST数据库。我目前正在进行第二次尝试,这是受这篇博文的启发:https://jamesmccaffrey.wordpress.com/2013/11/23/reading-the-mnist-data-set-with-c/。可悲的是,它仍然产生与我第一次尝试以奇怪的模式散布像素相同的废话:
我目前的阅读算法:
public static TrainingSet GenerateTrainingSet(FileInfo imagesFile, FileInfo labelsFile)
{
MnistImageView imageView = new MnistImageView();
imageView.Show();
TrainingSet trainingSet = new TrainingSet();
List<List<double>> labels = new List<List<double>>();
List<List<double>> images = new List<List<double>>();
using (BinaryReader brLabels = new BinaryReader(new FileStream(labelsFile.FullName, FileMode.Open)))
{
using (BinaryReader brImages = new BinaryReader(new FileStream(imagesFile.FullName, FileMode.Open)))
{
int magic1 = brImages.ReadBigInt32(); //Reading as BigEndian
int numImages = brImages.ReadBigInt32();
int numRows = brImages.ReadBigInt32();
int numCols = brImages.ReadBigInt32();
int magic2 = brLabels.ReadBigInt32();
int numLabels = brLabels.ReadBigInt32();
byte[] pixels = new byte[numRows * numCols];
// each image
for (int imageCounter = 0; imageCounter < numImages; imageCounter++)
{
List<double> imageInput = new List<double>();
List<double> exspectedOutput = new List<double>();
for (int i = 0; i < 10; i++) //generate empty exspected output
exspectedOutput.Add(0);
//read image
for (int p = 0; p < pixels.Length; p++)
{
byte b = brImages.ReadByte();
pixels[p] = b;
imageInput.Add(b / 255.0f); //scale in 0 to 1 range
}
//read label
byte lbl = brLabels.ReadByte();
exspectedOutput[lbl] = 1; //modify exspected output
labels.Add(exspectedOutput);
images.Add(imageInput);
//Debug view showing parsed image.......................
Bitmap image = new Bitmap(numCols, numRows);
for (int y = 0; y < numRows; y++)
{
for (int x = 0; x < numCols; x++)
{
image.SetPixel(x, y, Color.FromArgb(255 - pixels[x * y], 255 - pixels[x * y], 255 - pixels[x * y])); //invert colors to have 0,0,0 be white as specified by mnist
}
}
imageView.SetImage(image);
imageView.Refresh();
//.......................................................
}
brImages.Close();
brLabels.Close();
}
}
for (int i = 0; i < images.Count; i++)
{
trainingSet.data.Add(images[i], labels[i]);
}
return trainingSet;
}
所有图像都会产生如上所示的图案。它从来都不是完全相同的图案,但似乎总是有像素 "pulled" 到右下角。
我就是这样做的:
public static class MnistReader
{
private const string TrainImages = "mnist/train-images.idx3-ubyte";
private const string TrainLabels = "mnist/train-labels.idx1-ubyte";
private const string TestImages = "mnist/t10k-images.idx3-ubyte";
private const string TestLabels = "mnist/t10k-labels.idx1-ubyte";
public static IEnumerable<Image> ReadTrainingData()
{
foreach (var item in Read(TrainImages, TrainLabels))
{
yield return item;
}
}
public static IEnumerable<Image> ReadTestData()
{
foreach (var item in Read(TestImages, TestLabels))
{
yield return item;
}
}
private static IEnumerable<Image> Read(string imagesPath, string labelsPath)
{
BinaryReader labels = new BinaryReader(new FileStream(labelsPath, FileMode.Open));
BinaryReader images = new BinaryReader(new FileStream(imagesPath, FileMode.Open));
int magicNumber = images.ReadBigInt32();
int numberOfImages = images.ReadBigInt32();
int width = images.ReadBigInt32();
int height = images.ReadBigInt32();
int magicLabel = labels.ReadBigInt32();
int numberOfLabels = labels.ReadBigInt32();
for (int i = 0; i < numberOfImages; i++)
{
var bytes = images.ReadBytes(width * height);
var arr = new byte[height, width];
arr.ForEach((j,k) => arr[j, k] = bytes[j * height + k]);
yield return new Image()
{
Data = arr,
Label = labels.ReadByte()
};
}
}
}
Image
class:
public class Image
{
public byte Label { get; set; }
public byte[,] Data { get; set; }
}
一些扩展方法:
public static class Extensions
{
public static int ReadBigInt32(this BinaryReader br)
{
var bytes = br.ReadBytes(sizeof(Int32));
if (BitConverter.IsLittleEndian) Array.Reverse(bytes);
return BitConverter.ToInt32(bytes, 0);
}
public static void ForEach<T>(this T[,] source, Action<int, int> action)
{
for (int w = 0; w < source.GetLength(0); w++)
{
for (int h = 0; h < source.GetLength(1); h++)
{
action(w, h);
}
}
}
}
用法:
foreach (var image in MnistReader.ReadTrainingData())
{
//use image here
}
或
foreach (var image in MnistReader.ReadTestData())
{
//use image here
}
为什么不使用 nuget 包:
- MNIST.IO只是一个数据读取器(免责声明:我的包裹)
- Accord.DataSets包含类下载和解析MNIST、News20、Iris等机器学习数据集。这个包是 Accord.NET 框架的一部分。