有效地将System.Single[]转换为numpy数组
Efficiently convert System.Single[,] to numpy array
使用 Python 3.6 和 Python 用于 dotNET/pythonnet 我设法获得了一个图像数组。这是类型 System.Single[]
我想将它转换成一个 numpy 数组，这样我就可以在 Python 中用它做一些事情。我已经设置了一个函数来遍历该数组并将其按元素转换 - 但是我可以使用更明智（更快）的东西吗？
def MeasurementArrayToNumpy(TwoDArray):
hBound = TwoDArray.GetUpperBound(0)
vBound = TwoDArray.GetUpperBound(1)
resultArray = np.zeros([hBound, vBound])
for c in range(TwoDArray.GetUpperBound(0)):
for r in range(TwoDArray.GetUpperBound(1)):
resultArray[c,r] = TwoDArray[c,r]
return resultArray
@denfromufa - 这非常有用 link。
建议使用 Marshal.Copy 或 np.frombuffer 进行直接内存复制。我无法使 Marshal.Copy 版本正常工作 - 需要一些恶作剧才能将二维数组与 Marshal 一起使用，并且以某种方式改变了数组的内容 - 但 np.frombuffer 版本似乎适用于我将 3296*2471 阵列的完成时间减少了 ~16000 倍（~25s -> ~1.50ms）。这对我来说已经足够了
该方法需要更多导入，因此我将它们包含在下面的代码片段中
import ctypes
from System.Runtime.InteropServices import GCHandle, GCHandleType
def SingleToNumpyFromBuffer(TwoDArray):
src_hndl = GCHandle.Alloc(TwoDArray, GCHandleType.Pinned)
try:
src_ptr = src_hndl.AddrOfPinnedObject().ToInt32()
bufType = ctypes.c_float*len(TwoDArray)
cbuf = bufType.from_address(src_ptr)
resultArray = np.frombuffer(cbuf, dtype=cbuf._type_)
finally:
if src_hndl.IsAllocated: src_hndl.Free()
return resultArray
我修改了 函数，以便它可以用于 System.Int32[] 类型的 RGB 图像。然后将生成的 numpy 数组重新整形，以便可以在 opencv window
中显示图像
def net2Numpy(net_img,width,height):
src_hndl = GCHandle.Alloc(net_img, GCHandleType.Pinned)
try:
src_ptr = src_hndl.AddrOfPinnedObject().ToInt32()
bufType = ctypes.c_int*len(net_img)
cbuf = bufType.from_address(src_ptr)
resultArray = np.frombuffer(cbuf, dtype=cbuf._type_)
finally:
if src_hndl.IsAllocated: src_hndl.Free()
resultArray = resultArray.astype(dtype=np.uint8)
resultArray = resultArray.reshape((height,width,3),order='C')
return resultArray
继 denfromufa's link 之后，我认为 Robert McLeod 提供了最佳解决方案。他还指出了使用 np.frombuffer:
的缺点
one can do a zero-copy with np.frombuffer but then you have a mess of memory manged both by Python's garbage collector and C#'s garbage collector.
Robert McLeod 在 github 期中的摘录：
import numpy as np
import ctypes
import clr, System
from System import Array, Int32
from System.Runtime.InteropServices import GCHandle, GCHandleType
_MAP_NP_NET = {
np.dtype('float32'): System.Single,
np.dtype('float64'): System.Double,
np.dtype('int8') : System.SByte,
np.dtype('int16') : System.Int16,
np.dtype('int32') : System.Int32,
np.dtype('int64') : System.Int64,
np.dtype('uint8') : System.Byte,
np.dtype('uint16') : System.UInt16,
np.dtype('uint32') : System.UInt32,
np.dtype('uint64') : System.UInt64,
np.dtype('bool') : System.Boolean,
}
_MAP_NET_NP = {
'Single' : np.dtype('float32'),
'Double' : np.dtype('float64'),
'SByte' : np.dtype('int8'),
'Int16' : np.dtype('int16'),
'Int32' : np.dtype('int32'),
'Int64' : np.dtype('int64'),
'Byte' : np.dtype('uint8'),
'UInt16' : np.dtype('uint16'),
'UInt32' : np.dtype('uint32'),
'UInt64' : np.dtype('uint64'),
'Boolean': np.dtype('bool'),
}
def asNumpyArray(netArray):
'''
Given a CLR `System.Array` returns a `numpy.ndarray`. See _MAP_NET_NP for
the mapping of CLR types to Numpy dtypes.
'''
dims = np.empty(netArray.Rank, dtype=int)
for I in range(netArray.Rank):
dims[I] = netArray.GetLength(I)
netType = netArray.GetType().GetElementType().Name
try:
npArray = np.empty(dims, order='C', dtype=_MAP_NET_NP[netType])
except KeyError:
raise NotImplementedError("asNumpyArray does not yet support System type {}".format(netType) )
try: # Memmove
sourceHandle = GCHandle.Alloc(netArray, GCHandleType.Pinned)
sourcePtr = sourceHandle.AddrOfPinnedObject().ToInt64()
destPtr = npArray.__array_interface__['data'][0]
ctypes.memmove(destPtr, sourcePtr, npArray.nbytes)
finally:
if sourceHandle.IsAllocated: sourceHandle.Free()
return npArray
def asNetArray(npArray):
'''
Given a `numpy.ndarray` returns a CLR `System.Array`. See _MAP_NP_NET for
the mapping of Numpy dtypes to CLR types.
Note: `complex64` and `complex128` arrays are converted to `float32`
and `float64` arrays respectively with shape [m,n,...] -> [m,n,...,2]
'''
dims = npArray.shape
dtype = npArray.dtype
# For complex arrays, we must make a view of the array as its corresponding
# float type.
if dtype == np.complex64:
dtype = np.dtype('float32')
dims.append(2)
npArray = npArray.view(np.float32).reshape(dims)
elif dtype == np.complex128:
dtype = np.dtype('float64')
dims.append(2)
npArray = npArray.view(np.float64).reshape(dims)
netDims = Array.CreateInstance(Int32, npArray.ndim)
for I in range(npArray.ndim):
netDims[I] = Int32(dims[I])
if not npArray.flags.c_contiguous:
npArray = npArray.copy(order='C')
assert npArray.flags.c_contiguous
try:
netArray = Array.CreateInstance(_MAP_NP_NET[dtype], netDims)
except KeyError:
raise NotImplementedError("asNetArray does not yet support dtype {}".format(dtype))
try: # Memmove
destHandle = GCHandle.Alloc(netArray, GCHandleType.Pinned)
sourcePtr = npArray.__array_interface__['data'][0]
destPtr = destHandle.AddrOfPinnedObject().ToInt64()
ctypes.memmove(destPtr, sourcePtr, npArray.nbytes)
finally:
if destHandle.IsAllocated: destHandle.Free()
return netArray
if __name__ == '__main__':
from time import perf_counter
import matplotlib.pyplot as plt
import psutil
tries = 1000
foo = np.full([1024,1024], 2.5, dtype='float32')
netMem = np.zeros(tries)
t_asNet = np.zeros(tries)
netFoo = asNetArray( foo ) # Lazy loading makes the first iteration very slow
for I in range(tries):
t0 = perf_counter()
netFoo = asNetArray( foo )
t_asNet[I] = perf_counter() - t0
netMem[I] = psutil.virtual_memory().free / 2.0**20
t_asNumpy = np.zeros(tries)
numpyMem = np.zeros(tries)
unNetFoo = asNumpyArray( netFoo ) # Lazy loading makes the first iteration very slow
for I in range(tries):
t0 = perf_counter()
unNetFoo = asNumpyArray( netFoo )
t_asNumpy[I] = perf_counter() - t0
numpyMem[I] = psutil.virtual_memory().free / 2.0**20
# Convert times to milliseconds
t_asNet *= 1000
t_asNumpy *= 1000
np.testing.assert_array_almost_equal( unNetFoo, foo )
print( "Numpy to .NET converted {} bytes in {:.3f} +/- {:.3f} ms (mean: {:.1f} ns/ele)".format( \
foo.nbytes, t_asNet.mean(), t_asNet.std(), t_asNet.mean()/foo.size*1e6 ) )
print( ".NET to Numpy converted {} bytes in {:.3f} +/- {:.3f} ms (mean: {:.1f} ns/ele)".format( \
foo.nbytes, t_asNumpy.mean(), t_asNumpy.std(), t_asNumpy.mean()/foo.size*1e6 ) )
plt.figure()
plt.plot(np.arange(tries), netMem, '-', label='asNetArray')
plt.plot(np.arange(tries), numpyMem, '-', label='asNumpyArray')
plt.legend(loc='best')
plt.ylabel('Free memory (MB)')
plt.xlabel('Iteration')
plt.show(block=True)
还值得注意的是，pythonnet 有一个新的实验性功能，看起来很有希望：Codecs。仅当您从源代码构建并设法找出文档时才相关：
使用 Python 3.6 和 Python 用于 dotNET/pythonnet 我设法获得了一个图像数组。这是类型 System.Single[]
我想将它转换成一个 numpy 数组，这样我就可以在 Python 中用它做一些事情。我已经设置了一个函数来遍历该数组并将其按元素转换 - 但是我可以使用更明智（更快）的东西吗？
def MeasurementArrayToNumpy(TwoDArray):
hBound = TwoDArray.GetUpperBound(0)
vBound = TwoDArray.GetUpperBound(1)
resultArray = np.zeros([hBound, vBound])
for c in range(TwoDArray.GetUpperBound(0)):
for r in range(TwoDArray.GetUpperBound(1)):
resultArray[c,r] = TwoDArray[c,r]
return resultArray
@denfromufa - 这非常有用 link。
建议使用 Marshal.Copy 或 np.frombuffer 进行直接内存复制。我无法使 Marshal.Copy 版本正常工作 - 需要一些恶作剧才能将二维数组与 Marshal 一起使用，并且以某种方式改变了数组的内容 - 但 np.frombuffer 版本似乎适用于我将 3296*2471 阵列的完成时间减少了 ~16000 倍（~25s -> ~1.50ms）。这对我来说已经足够了
该方法需要更多导入，因此我将它们包含在下面的代码片段中
import ctypes
from System.Runtime.InteropServices import GCHandle, GCHandleType
def SingleToNumpyFromBuffer(TwoDArray):
src_hndl = GCHandle.Alloc(TwoDArray, GCHandleType.Pinned)
try:
src_ptr = src_hndl.AddrOfPinnedObject().ToInt32()
bufType = ctypes.c_float*len(TwoDArray)
cbuf = bufType.from_address(src_ptr)
resultArray = np.frombuffer(cbuf, dtype=cbuf._type_)
finally:
if src_hndl.IsAllocated: src_hndl.Free()
return resultArray
我修改了
def net2Numpy(net_img,width,height):
src_hndl = GCHandle.Alloc(net_img, GCHandleType.Pinned)
try:
src_ptr = src_hndl.AddrOfPinnedObject().ToInt32()
bufType = ctypes.c_int*len(net_img)
cbuf = bufType.from_address(src_ptr)
resultArray = np.frombuffer(cbuf, dtype=cbuf._type_)
finally:
if src_hndl.IsAllocated: src_hndl.Free()
resultArray = resultArray.astype(dtype=np.uint8)
resultArray = resultArray.reshape((height,width,3),order='C')
return resultArray
继 denfromufa's link 之后，我认为 Robert McLeod 提供了最佳解决方案。他还指出了使用 np.frombuffer:
one can do a zero-copy with np.frombuffer but then you have a mess of memory manged both by Python's garbage collector and C#'s garbage collector.
Robert McLeod 在 github 期中的摘录：
import numpy as np
import ctypes
import clr, System
from System import Array, Int32
from System.Runtime.InteropServices import GCHandle, GCHandleType
_MAP_NP_NET = {
np.dtype('float32'): System.Single,
np.dtype('float64'): System.Double,
np.dtype('int8') : System.SByte,
np.dtype('int16') : System.Int16,
np.dtype('int32') : System.Int32,
np.dtype('int64') : System.Int64,
np.dtype('uint8') : System.Byte,
np.dtype('uint16') : System.UInt16,
np.dtype('uint32') : System.UInt32,
np.dtype('uint64') : System.UInt64,
np.dtype('bool') : System.Boolean,
}
_MAP_NET_NP = {
'Single' : np.dtype('float32'),
'Double' : np.dtype('float64'),
'SByte' : np.dtype('int8'),
'Int16' : np.dtype('int16'),
'Int32' : np.dtype('int32'),
'Int64' : np.dtype('int64'),
'Byte' : np.dtype('uint8'),
'UInt16' : np.dtype('uint16'),
'UInt32' : np.dtype('uint32'),
'UInt64' : np.dtype('uint64'),
'Boolean': np.dtype('bool'),
}
def asNumpyArray(netArray):
'''
Given a CLR `System.Array` returns a `numpy.ndarray`. See _MAP_NET_NP for
the mapping of CLR types to Numpy dtypes.
'''
dims = np.empty(netArray.Rank, dtype=int)
for I in range(netArray.Rank):
dims[I] = netArray.GetLength(I)
netType = netArray.GetType().GetElementType().Name
try:
npArray = np.empty(dims, order='C', dtype=_MAP_NET_NP[netType])
except KeyError:
raise NotImplementedError("asNumpyArray does not yet support System type {}".format(netType) )
try: # Memmove
sourceHandle = GCHandle.Alloc(netArray, GCHandleType.Pinned)
sourcePtr = sourceHandle.AddrOfPinnedObject().ToInt64()
destPtr = npArray.__array_interface__['data'][0]
ctypes.memmove(destPtr, sourcePtr, npArray.nbytes)
finally:
if sourceHandle.IsAllocated: sourceHandle.Free()
return npArray
def asNetArray(npArray):
'''
Given a `numpy.ndarray` returns a CLR `System.Array`. See _MAP_NP_NET for
the mapping of Numpy dtypes to CLR types.
Note: `complex64` and `complex128` arrays are converted to `float32`
and `float64` arrays respectively with shape [m,n,...] -> [m,n,...,2]
'''
dims = npArray.shape
dtype = npArray.dtype
# For complex arrays, we must make a view of the array as its corresponding
# float type.
if dtype == np.complex64:
dtype = np.dtype('float32')
dims.append(2)
npArray = npArray.view(np.float32).reshape(dims)
elif dtype == np.complex128:
dtype = np.dtype('float64')
dims.append(2)
npArray = npArray.view(np.float64).reshape(dims)
netDims = Array.CreateInstance(Int32, npArray.ndim)
for I in range(npArray.ndim):
netDims[I] = Int32(dims[I])
if not npArray.flags.c_contiguous:
npArray = npArray.copy(order='C')
assert npArray.flags.c_contiguous
try:
netArray = Array.CreateInstance(_MAP_NP_NET[dtype], netDims)
except KeyError:
raise NotImplementedError("asNetArray does not yet support dtype {}".format(dtype))
try: # Memmove
destHandle = GCHandle.Alloc(netArray, GCHandleType.Pinned)
sourcePtr = npArray.__array_interface__['data'][0]
destPtr = destHandle.AddrOfPinnedObject().ToInt64()
ctypes.memmove(destPtr, sourcePtr, npArray.nbytes)
finally:
if destHandle.IsAllocated: destHandle.Free()
return netArray
if __name__ == '__main__':
from time import perf_counter
import matplotlib.pyplot as plt
import psutil
tries = 1000
foo = np.full([1024,1024], 2.5, dtype='float32')
netMem = np.zeros(tries)
t_asNet = np.zeros(tries)
netFoo = asNetArray( foo ) # Lazy loading makes the first iteration very slow
for I in range(tries):
t0 = perf_counter()
netFoo = asNetArray( foo )
t_asNet[I] = perf_counter() - t0
netMem[I] = psutil.virtual_memory().free / 2.0**20
t_asNumpy = np.zeros(tries)
numpyMem = np.zeros(tries)
unNetFoo = asNumpyArray( netFoo ) # Lazy loading makes the first iteration very slow
for I in range(tries):
t0 = perf_counter()
unNetFoo = asNumpyArray( netFoo )
t_asNumpy[I] = perf_counter() - t0
numpyMem[I] = psutil.virtual_memory().free / 2.0**20
# Convert times to milliseconds
t_asNet *= 1000
t_asNumpy *= 1000
np.testing.assert_array_almost_equal( unNetFoo, foo )
print( "Numpy to .NET converted {} bytes in {:.3f} +/- {:.3f} ms (mean: {:.1f} ns/ele)".format( \
foo.nbytes, t_asNet.mean(), t_asNet.std(), t_asNet.mean()/foo.size*1e6 ) )
print( ".NET to Numpy converted {} bytes in {:.3f} +/- {:.3f} ms (mean: {:.1f} ns/ele)".format( \
foo.nbytes, t_asNumpy.mean(), t_asNumpy.std(), t_asNumpy.mean()/foo.size*1e6 ) )
plt.figure()
plt.plot(np.arange(tries), netMem, '-', label='asNetArray')
plt.plot(np.arange(tries), numpyMem, '-', label='asNumpyArray')
plt.legend(loc='best')
plt.ylabel('Free memory (MB)')
plt.xlabel('Iteration')
plt.show(block=True)
还值得注意的是，pythonnet 有一个新的实验性功能，看起来很有希望：Codecs。仅当您从源代码构建并设法找出文档时才相关：