从 NumPy 数组到 Mat (OpenCV) 的 C++ 转换
C++ conversion from NumPy array to Mat (OpenCV)
我正在围绕 ArUco 增强现实库（基于 OpenCV）编写一个薄包装器。我尝试构建的界面非常简单：
- Python 将图像传递给 C++ 代码；
- C++ 代码检测标记和 returns 它们的位置以及其他信息 Python 作为字典的元组。
但是，我不知道如何在 Python 中表示图像以将其传递给 C++。对于 GUI 和摄像头管理，我将使用 PyQt，所以最初它将是 QImage，但我不能简单地将它传递给 OpenCV（或者我可以？）。起初，我尝试使用嵌套元组来表示每个像素的行、列和颜色，所以我最终得到了这个示例代码：
using namespace cv;
namespace py = boost::python;
void display(py::tuple pix)
{
/*
Receive image from Python and display it.
*/
Mat img(py::len(pix), py::len(pix[0]), CV_8UC3, Scalar(0, 0, 255));
for (int y = 0; y < py::len(pix); y++)
for (int x = 0; x < py::len(pix[y]); x++)
{
Vec3b rgb;
for (int i = 0; i < 3; i++)
rgb[i] = py::extract<int>(pix[y][x][i]);
img.at<Vec3b>(Point(x, y)) = rgb;
}
imshow("Image", img);
waitKey(0);
}
BOOST_PYTHON_MODULE(aruco)
{
py::def("display", display);
}
事实证明它非常慢（单帧几秒钟），所以我去谷歌搜索并找到了应该更快的解决方案：使用 NumPy 数组，所以代码看起来像这样：
void display(py::object array)
{
Mat img;
// ... some magic here to convert NumPy array to Mat ...
imshow("Image", img);
waitKey(0);
}
但是，我不知道如何将 NumPy 数组（在 C++ 级别只是一个 Python 对象）转换为 OpenCV Mat。如果有任何帮助，我将不胜感激。
或者，也许并不是真的需要 NumPy，所以我可以将 QImage Python 对象直接传递给 C++ 层吗？或者也许有不同的方法来解决这个问题？任何建议表示赞赏！
根据您的情况，最佳解决方案是为 cv::Mat 对象使用自定义 boost::python 转换器。 OpenCV 有 Python 包装器，当你使用这个包装器时，你正在对 Numpy 数组进行操作——你甚至不需要知道这些数组在 "crossing the c++ <-> python border" 时被转换为 cv::Mat 对象。为简单类型编写这样的转换器非常容易，但是为 cv::Mat 创建转换器并不简单。幸运的是，其他人已经这样做了 - here is version for OpenCV 2.x and here for 3.x. If you are not familiar with boost::python converters, this article 应该可以帮助你。
希望对您有所帮助，如果您有任何问题，请告诉我们。
可选，如果你不喜欢使用包装器，想使用原生的python扩展模块，你可以这样做。
python3:
my_image = cv.imread("my_image.jpg", 1) # reads colorfull image in python
dims = my_image.shape # get image shape (h, w, c)
my_image = my_image.ravel() # flattens 3d array into 1d
cppextenionmodule.np_to_mat(dims, my_image)
c++:
static PyObject *np_to_mat(PyObject *self, PyObject *args){
PyObject *size;
PyArrayObject *image;
if (!PyArg_ParseTuple(args, "O!O!", &PyTuple_Type, &size, &PyArray_Type, &image)) {
return NULL;
}
int rows = PyLong_AsLong(PyTuple_GetItem(size ,0));
int cols = PyLong_AsLong(PyTuple_GetItem(size ,1));
int nchannels = PyLong_AsLong(PyTuple_GetItem(size ,2));
char my_arr[rows * nchannels * cols];
for(size_t length = 0; length<(rows * nchannels * cols); length++){
my_arr[length] = (*(char *)PyArray_GETPTR1(image, length));
}
cv::Mat my_img = cv::Mat(cv::Size(cols, rows), CV_8UC3, &my_arr);
... whatever with the image
}
我为不知道有 Boost Numpy 模块的人写了这个例子。您可以看到如何将 Mat 转换为 NDArray，反之亦然。它会让您了解转换 ndarray 的方式。
#define BOOST_PYTHON_STATIC_LIB
#define BOOST_LIB_NAME "boost_numpy35"
//#include <boost/config/auto_link.hpp>
#include <boost/python.hpp>
#include <boost/python/numpy.hpp>
#include <iostream>
#include <opencv2/opencv.hpp>
namespace py = boost::python;
namespace np = boost::python::numpy;
void Init() {
// set your python location.
wchar_t str[] = L"D:\Anaconda3\envs\tensorflow_vision";
Py_SetPythonHome(str);
Py_Initialize();
np::initialize();
}
np::ndarray ConvertMatToNDArray(const cv::Mat& mat) {
py::tuple shape = py::make_tuple(mat.rows, mat.cols, mat.channels());
py::tuple stride = py::make_tuple(mat.channels() * mat.cols * sizeof(uchar), mat.channels() * sizeof(uchar), sizeof(uchar));
np::dtype dt = np::dtype::get_builtin<uchar>();
np::ndarray ndImg = np::from_data(mat.data, dt, shape, stride, py::object());
return ndImg;
}
cv::Mat ConvertNDArrayToMat(const np::ndarray& ndarr) {
//int length = ndarr.get_nd(); // get_nd() returns num of dimensions. this is used as a length, but we don't need to use in this case. because we know that image has 3 dimensions.
const Py_intptr_t* shape = ndarr.get_shape(); // get_shape() returns Py_intptr_t* which we can get the size of n-th dimension of the ndarray.
char* dtype_str = py::extract<char *>(py::str(ndarr.get_dtype()));
// variables for creating Mat object
int rows = shape[0];
int cols = shape[1];
int channel = shape[2];
int depth;
// you should find proper type for c++. in this case we use 'CV_8UC3' image, so we need to create 'uchar' type Mat.
if (!strcmp(dtype_str, "uint8")) {
depth = CV_8U;
}
else {
std::cout << "wrong dtype error" << std::endl;
return cv::Mat();
}
int type = CV_MAKETYPE(depth, channel); // CV_8UC3
cv::Mat mat = cv::Mat(rows, cols, type);
memcpy(mat.data, ndarr.get_data(), sizeof(uchar) * rows * cols * channel);
return mat;
}
int main()
{
using namespace std;
try
{
// initialize boost python and numpy
Init();
// import module
py::object main_module = py::import("__main__");
py::object print = main_module.attr("__builtins__").attr("print"); // this is for printing python object
// get image
cv::Mat img;
img = cv::imread("Lenna.jpg", cv::IMREAD_COLOR);
if (img.empty())
{
std::cout << "can't getting image" << std::endl;
return -1;
}
// convert Mat to NDArray
cv::Mat cloneImg = img.clone(); // converting functions will access to same data between Mat and NDArray. so we should clone Mat object. This may important in your case.
np::ndarray ndImg = ConvertMatToNDArray(cloneImg);
// You can check if it's properly converted.
//print(ndImg);
// convert NDArray to Mat
cv::Mat matImg = ConvertNDArrayToMat(ndImg); // also you can convert ndarray to mat.
// add 10 brightness to converted image
for (int i = 0; i < matImg.rows; i++) {
for (int j = 0; j < matImg.cols; j++) {
for (int c = 0; c < matImg.channels(); c++) {
matImg.at<cv::Vec3b>(i, j)[c] += 10;
}
}
}
// show image
cv::imshow("original image", img);
cv::imshow("converted image", matImg);
cv::waitKey(0);
cv::destroyAllWindows();
}
catch (py::error_already_set&)
{
PyErr_Print();
system("pause");
}
system("pause");
return 0;
}
这是 afewthings/DomQ 答案的 pybind11 版本。我发现 pybind11 比 boost::python 更适合我的项目（两个库都很好）
// convert a cv::Mat to an np.array
py::array to_array(const cv::Mat& im) {
const ssize_t channels = im.channels();
const ssize_t height = im.rows;
const ssize_t width = im.cols;
const ssize_t dim = sizeof(uchar) * height * width * channels;
auto data = new uchar[dim];
std::copy(im.data, im.data + dim, data);
return py::array_t<uchar>(
py::buffer_info(
data,
sizeof(uchar), //itemsize
py::format_descriptor<uchar>::format(),
channels, // ndim
std::vector<ssize_t> { height, width, channels }, // shape
std::vector<ssize_t> { width * channels, channels, sizeof(uchar) } // strides
),
py::capsule(data, [](void* f){
// handle releasing data
delete[] reinterpret_cast<uchar*>(f);
})
);
}
// convert an np.array to a cv::Mat
cv::Mat from_array(const py::array& ar) {
if (!ar.dtype().is(py::dtype::of<uchar>())) {
std::cout << "ERROR unsupported dtype!" << std::endl;
return cv::Mat();
}
auto shape = ar.shape();
int rows = shape[0];
int cols = shape[1];
int channels = shape[2];
int type = CV_MAKETYPE(CV_8U, channels); // CV_8UC3
cv::Mat mat = cv::Mat(rows, cols, type);
memcpy(mat.data, ar.data(), sizeof(uchar) * rows * cols * channels);
return mat;
}
我正在围绕 ArUco 增强现实库（基于 OpenCV）编写一个薄包装器。我尝试构建的界面非常简单：
- Python 将图像传递给 C++ 代码；
- C++ 代码检测标记和 returns 它们的位置以及其他信息 Python 作为字典的元组。
但是，我不知道如何在 Python 中表示图像以将其传递给 C++。对于 GUI 和摄像头管理，我将使用 PyQt，所以最初它将是 QImage，但我不能简单地将它传递给 OpenCV（或者我可以？）。起初，我尝试使用嵌套元组来表示每个像素的行、列和颜色，所以我最终得到了这个示例代码：
using namespace cv;
namespace py = boost::python;
void display(py::tuple pix)
{
/*
Receive image from Python and display it.
*/
Mat img(py::len(pix), py::len(pix[0]), CV_8UC3, Scalar(0, 0, 255));
for (int y = 0; y < py::len(pix); y++)
for (int x = 0; x < py::len(pix[y]); x++)
{
Vec3b rgb;
for (int i = 0; i < 3; i++)
rgb[i] = py::extract<int>(pix[y][x][i]);
img.at<Vec3b>(Point(x, y)) = rgb;
}
imshow("Image", img);
waitKey(0);
}
BOOST_PYTHON_MODULE(aruco)
{
py::def("display", display);
}
事实证明它非常慢（单帧几秒钟），所以我去谷歌搜索并找到了应该更快的解决方案：使用 NumPy 数组，所以代码看起来像这样：
void display(py::object array)
{
Mat img;
// ... some magic here to convert NumPy array to Mat ...
imshow("Image", img);
waitKey(0);
}
但是，我不知道如何将 NumPy 数组（在 C++ 级别只是一个 Python 对象）转换为 OpenCV Mat。如果有任何帮助，我将不胜感激。
或者，也许并不是真的需要 NumPy，所以我可以将 QImage Python 对象直接传递给 C++ 层吗？或者也许有不同的方法来解决这个问题？任何建议表示赞赏！
根据您的情况，最佳解决方案是为 cv::Mat 对象使用自定义 boost::python 转换器。 OpenCV 有 Python 包装器，当你使用这个包装器时，你正在对 Numpy 数组进行操作——你甚至不需要知道这些数组在 "crossing the c++ <-> python border" 时被转换为 cv::Mat 对象。为简单类型编写这样的转换器非常容易，但是为 cv::Mat 创建转换器并不简单。幸运的是，其他人已经这样做了 - here is version for OpenCV 2.x and here for 3.x. If you are not familiar with boost::python converters, this article 应该可以帮助你。
希望对您有所帮助，如果您有任何问题，请告诉我们。
可选，如果你不喜欢使用包装器，想使用原生的python扩展模块，你可以这样做。
python3:
my_image = cv.imread("my_image.jpg", 1) # reads colorfull image in python
dims = my_image.shape # get image shape (h, w, c)
my_image = my_image.ravel() # flattens 3d array into 1d
cppextenionmodule.np_to_mat(dims, my_image)
c++:
static PyObject *np_to_mat(PyObject *self, PyObject *args){
PyObject *size;
PyArrayObject *image;
if (!PyArg_ParseTuple(args, "O!O!", &PyTuple_Type, &size, &PyArray_Type, &image)) {
return NULL;
}
int rows = PyLong_AsLong(PyTuple_GetItem(size ,0));
int cols = PyLong_AsLong(PyTuple_GetItem(size ,1));
int nchannels = PyLong_AsLong(PyTuple_GetItem(size ,2));
char my_arr[rows * nchannels * cols];
for(size_t length = 0; length<(rows * nchannels * cols); length++){
my_arr[length] = (*(char *)PyArray_GETPTR1(image, length));
}
cv::Mat my_img = cv::Mat(cv::Size(cols, rows), CV_8UC3, &my_arr);
... whatever with the image
}
我为不知道有 Boost Numpy 模块的人写了这个例子。您可以看到如何将 Mat 转换为 NDArray，反之亦然。它会让您了解转换 ndarray 的方式。
#define BOOST_PYTHON_STATIC_LIB
#define BOOST_LIB_NAME "boost_numpy35"
//#include <boost/config/auto_link.hpp>
#include <boost/python.hpp>
#include <boost/python/numpy.hpp>
#include <iostream>
#include <opencv2/opencv.hpp>
namespace py = boost::python;
namespace np = boost::python::numpy;
void Init() {
// set your python location.
wchar_t str[] = L"D:\Anaconda3\envs\tensorflow_vision";
Py_SetPythonHome(str);
Py_Initialize();
np::initialize();
}
np::ndarray ConvertMatToNDArray(const cv::Mat& mat) {
py::tuple shape = py::make_tuple(mat.rows, mat.cols, mat.channels());
py::tuple stride = py::make_tuple(mat.channels() * mat.cols * sizeof(uchar), mat.channels() * sizeof(uchar), sizeof(uchar));
np::dtype dt = np::dtype::get_builtin<uchar>();
np::ndarray ndImg = np::from_data(mat.data, dt, shape, stride, py::object());
return ndImg;
}
cv::Mat ConvertNDArrayToMat(const np::ndarray& ndarr) {
//int length = ndarr.get_nd(); // get_nd() returns num of dimensions. this is used as a length, but we don't need to use in this case. because we know that image has 3 dimensions.
const Py_intptr_t* shape = ndarr.get_shape(); // get_shape() returns Py_intptr_t* which we can get the size of n-th dimension of the ndarray.
char* dtype_str = py::extract<char *>(py::str(ndarr.get_dtype()));
// variables for creating Mat object
int rows = shape[0];
int cols = shape[1];
int channel = shape[2];
int depth;
// you should find proper type for c++. in this case we use 'CV_8UC3' image, so we need to create 'uchar' type Mat.
if (!strcmp(dtype_str, "uint8")) {
depth = CV_8U;
}
else {
std::cout << "wrong dtype error" << std::endl;
return cv::Mat();
}
int type = CV_MAKETYPE(depth, channel); // CV_8UC3
cv::Mat mat = cv::Mat(rows, cols, type);
memcpy(mat.data, ndarr.get_data(), sizeof(uchar) * rows * cols * channel);
return mat;
}
int main()
{
using namespace std;
try
{
// initialize boost python and numpy
Init();
// import module
py::object main_module = py::import("__main__");
py::object print = main_module.attr("__builtins__").attr("print"); // this is for printing python object
// get image
cv::Mat img;
img = cv::imread("Lenna.jpg", cv::IMREAD_COLOR);
if (img.empty())
{
std::cout << "can't getting image" << std::endl;
return -1;
}
// convert Mat to NDArray
cv::Mat cloneImg = img.clone(); // converting functions will access to same data between Mat and NDArray. so we should clone Mat object. This may important in your case.
np::ndarray ndImg = ConvertMatToNDArray(cloneImg);
// You can check if it's properly converted.
//print(ndImg);
// convert NDArray to Mat
cv::Mat matImg = ConvertNDArrayToMat(ndImg); // also you can convert ndarray to mat.
// add 10 brightness to converted image
for (int i = 0; i < matImg.rows; i++) {
for (int j = 0; j < matImg.cols; j++) {
for (int c = 0; c < matImg.channels(); c++) {
matImg.at<cv::Vec3b>(i, j)[c] += 10;
}
}
}
// show image
cv::imshow("original image", img);
cv::imshow("converted image", matImg);
cv::waitKey(0);
cv::destroyAllWindows();
}
catch (py::error_already_set&)
{
PyErr_Print();
system("pause");
}
system("pause");
return 0;
}
这是 afewthings/DomQ 答案的 pybind11 版本。我发现 pybind11 比 boost::python 更适合我的项目（两个库都很好）
// convert a cv::Mat to an np.array
py::array to_array(const cv::Mat& im) {
const ssize_t channels = im.channels();
const ssize_t height = im.rows;
const ssize_t width = im.cols;
const ssize_t dim = sizeof(uchar) * height * width * channels;
auto data = new uchar[dim];
std::copy(im.data, im.data + dim, data);
return py::array_t<uchar>(
py::buffer_info(
data,
sizeof(uchar), //itemsize
py::format_descriptor<uchar>::format(),
channels, // ndim
std::vector<ssize_t> { height, width, channels }, // shape
std::vector<ssize_t> { width * channels, channels, sizeof(uchar) } // strides
),
py::capsule(data, [](void* f){
// handle releasing data
delete[] reinterpret_cast<uchar*>(f);
})
);
}
// convert an np.array to a cv::Mat
cv::Mat from_array(const py::array& ar) {
if (!ar.dtype().is(py::dtype::of<uchar>())) {
std::cout << "ERROR unsupported dtype!" << std::endl;
return cv::Mat();
}
auto shape = ar.shape();
int rows = shape[0];
int cols = shape[1];
int channels = shape[2];
int type = CV_MAKETYPE(CV_8U, channels); // CV_8UC3
cv::Mat mat = cv::Mat(rows, cols, type);
memcpy(mat.data, ar.data(), sizeof(uchar) * rows * cols * channels);
return mat;
}