执行创建数组和图形的 python 脚本时出现内存问题
Memory problem executing python script that creates arrays and figures
我会先描述脚本，然后发布代码和重现问题的方法，在我的机器上可能不是每台机器都是这样。
所以我有一个脚本可以构建具有特定属性的对象、云，并将数据导出到文件，该文件是名为 SHDOM 的软件的输入，该工具分析辐射属性并可在以下 link.
中进一步阅读
剧本有点长，提前见谅：
import numpy as np
from itertools import product
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from mpl_toolkits.mplot3d import Axes3D
import gc
def writeDomainFile(type, file_name, temp_vec,
lwc, r_effective, dx, dy,
height_vec):
if type == 2:
assert list(lwc.shape) == list(r_effective.shape), "when the file type is 2 the shapes of lwc and r_effective shold be the same"
x_lwc, y_lwc, z_lwc = lwc.shape
x_r_effetive, y_r_effetive, z_r_effetive = r_effective.shape
fp = open(file_name, 'w')
# write the file format
fp.write("{:}\n".format(type))
# write the number of x, y, z points
fp.write("{:} {:} {:}\n".format(x_lwc, y_lwc, z_lwc))
# write the spacing resolution in x and y
fp.write("{:} {:}\n".format(dx, dy))
# write the height vector
fp.write(" ".join(map(str,height_vec)) + "\n")
# write the temprature vector
fp.write(" ".join(map(str,temp_vec)) + "\n")
indices = product(range(x_lwc), range(y_lwc), range(z_lwc))
print_indices = product(range(1, x_lwc + 1), range(1, y_lwc + 1), range(1, z_lwc + 1))
if type == 1: # only lwc
for triplet, print_triplets in zip(indices, print_indices):
# t = tuple(1 + list(triplet))
if not lwc[triplet]:
fp.write("{:} {:} {:} {:}\n".format(*print_triplets, 0))
else:
fp.write("{:} {:} {:} {:6.4f}\n".format(*print_triplets, lwc[triplet]))
elif type == 2:
for triplet, print_triplets in zip(indices, print_indices):
# t = tuple(map(lambda x: x + 1, list(triplet)))
if not lwc[triplet]:
fp.write("{:} {:} {:} {:} {:}\n".format(*print_triplets, 0, 0))
else:
fp.write("{:} {:} {:} {:6.4f} {:4.2f}\n".format(*print_triplets, lwc[triplet], r_effective[triplet]))
fp.close()
return
def lapseRateTemp(base_temp, position):
temp = np.zeros(position.shape)
temp[0] = base_temp
temp[1:] = base_temp - 6.5 * position[1:]
return np.round(temp, 4)
def createCubicCloudDomain(cloud_tick, x_size, y_size,
dx, dy, dz, r_effective_option, lwc_option,
cloud_x_size, cloud_y_size):
# const
temp0 = 300 # kelvin
N_x = x_size / dx # number of x points
N_y = y_size / dy # number of y points
cld_base = 1.0 # the cloud always start from 1[km] above ground
x_center = N_x / 2
y_center = N_y / 2
prog_x = (cloud_x_size / 2) / dx
prog_y = (cloud_y_size / 2) / dy
cloud_x_west = int(x_center - prog_x)
cloud_x_east = int(x_center + prog_x)
cloud_y_south = int(y_center - prog_y)
cloud_y_north = int(y_center + prog_y)
cloud_x_vec = np.arange(cloud_x_west, cloud_x_east, 1)
cloud_y_vec = np.arange(cloud_y_south, cloud_y_north, 1)
for tick in cloud_tick: # cloud_tick might be a vector
cld_top = cld_base + tick
N_z = tick / dz # number of z points
cloud_z_vec = np.round(np.arange(cld_base, cld_top, dz), 4)
# temprature
temp_base = temp0 - 9.8 * cld_base
temp_vec = lapseRateTemp(temp_base, cloud_z_vec - cld_base,)
temp_cloud = np.tile(temp_vec,(int(temp_vec.shape[0]), int(temp_vec.shape[0]), 1))
temp_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
temp_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = temp_cloud
plotTemperatureGradient(temp_domain, 'test.png')
# del temp_cloud
# del temp_domain
# gc.collect()
for r in r_effective_option:
for l in lwc_option:
r_effective_cloud = np.full((cloud_x_vec.shape[0],
cloud_y_vec.shape[0],
cloud_z_vec.shape[0]),
r)
lwc_cloud = np.full((cloud_x_vec.shape[0],
cloud_y_vec.shape[0],
cloud_z_vec.shape[0]),
l)
r_effective_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
lwc_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
# the positions to enter the cloud data
lwc_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = lwc_cloud
r_effective_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = r_effective_cloud
writeDomainFile(2, "test.txt", temp_vec, lwc_domain, r_effective_domain, dx, dy, cloud_z_vec)
plotGeneratedCloud(lwc_domain, r_effective_domain)
return
def plotTemperatureGradient(temp_mat, file_name):
xs, ys, zs = temp_mat.shape
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
X, Y, Z = np.mgrid[:xs, :ys, :zs]
faltten_data = temp_mat.ravel().astype(np.float16)
color_map = np.zeros((faltten_data.shape[0], 4))
# map scalars to colors
minima = np.min(faltten_data[np.nonzero(faltten_data)])
maxima = np.max(faltten_data[np.nonzero(faltten_data)])
norm = matplotlib.colors.Normalize(vmin=minima, vmax=maxima, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap='jet')
rgba = mapper.to_rgba(faltten_data)
color_map[:,0:3] = rgba[:, 0:3]
color_map[:,3] = np.where(faltten_data > 0, 0.07, 0)
p = ax.scatter(X, Y, Z, c=color_map.astype(np.float16))
ax.set_xlabel('X position [Arb.]')
ax.set_ylabel('Y position [Arb.]')
ax.set_zlabel('Z position [Arb.]')
fig.colorbar(p)
# plt.title(title)
plt.savefig(file_name)
return
def plotGeneratedCloud(lwc, r_effective):
light_blue = np.array([0, 1, 0.7])
matrixPlotter(lwc, 'lwc.png',
'Liquid water content of the cloud',
light_blue)
# gc.collect()
light_pink = np.array([0.9, 0.6, 0.9])
matrixPlotter(r_effective, 'r_effective.png',
'Effective radius of the cloud',
light_pink)
return
def matrixPlotter(mat, file_name, title, c=np.array([0.2, 0.6, 0])):
xs, ys, zs = mat.shape
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
X, Y, Z = np.mgrid[:xs, :ys, :zs]
faltten_data = mat.ravel().astype(np.float16)
color_map = np.zeros((faltten_data.shape[0], 4))
color_map[:,0:3] = c
color_map[:,3] = np.where(faltten_data > 0, 0.07, 0)
ax.scatter(X, Y, Z, c=color_map.astype(np.float16))
ax.set_xlabel('X position [Arb.]')
ax.set_ylabel('Y position [Arb.]')
ax.set_zlabel('Z position [Arb.]')
plt.title(title)
plt.savefig(file_name)
return
# def main():
# return
if __name__ == '__main__':
# main()
createCubicCloudDomain([0.5], 2.02, 2.02, 0.01, 0.01, 0.01, [0.5], [1], 0.5, 0.5)
print("Done")
所以问题发生在：
- 我尝试导出所有 3 个图，调用以下块：
plotGeneratedCloud(lwc_domain, r_effective_domain) 或
temp_cloud = np.tile(temp_vec,(int(temp_vec.shape[0]), int(temp_vec.shape[0]), 1))
temp_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
temp_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = temp_cloud
plotTemperatureGradient(temp_domain, 'test.png')
- 当我尝试创建超过 1 个对象（云）时，调用函数时：
createCubicCloudDomain([0.5, 0.6], 2.02, 2.02, 0.01, 0.01, 0.01, [0.5, 0.4], [1, 0.3], 0.5, 0.5)
- 增加域大小，例如：
createCubicCloudDomain([0.5], 4.02, 4.02, 0.01, 0.01, 0.01, [0.5], [1], 0.5, 0.5)
我遇到的错误类型如下：
File
"C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\backends\backend_agg.py",
line 396, in draw
self.figure.draw(self.renderer) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\artist.py",
line 38, in draw_wrapper
return draw(artist, renderer, *args, **kwargs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\figure.py",
line 1735, in draw
mimage._draw_list_compositing_images( File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\image.py",
line 137, in _draw_list_compositing_images
a.draw(renderer) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\artist.py",
line 38, in draw_wrapper
return draw(artist, renderer, *args, **kwargs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\axes3d.py",
line 291, in draw
sorted(self.collections, File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\axes3d.py",
line 292, in
key=lambda col: col.do_3d_projection(renderer), File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\art3d.py",
line 538, in do_3d_projection
vxs, vys, vzs, vis = proj3d.proj_transform_clip(xs, ys, zs, renderer.M) File
"C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\proj3d.py",
line 214, in proj_transform_clip
vec = _vec_pad_ones(xs, ys, zs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\proj3d.py",
line 189, in _vec_pad_ones
return np.array([xs, ys, zs, np.ones_like(xs)]) MemoryError: Unable to allocate array with shape (4, 2040200) and data type float64
我知道问题是内存问题，所以我认为一旦完成删除数组可能会解决问题，所以我使用了 del 函数，甚至使用了以下问题中使用的垃圾收集器解决方案: How can I explicitly free memory in Python?
希望得到一些帮助
如果您不先.close() figure 对象，它会保留在内存中，并且它包含它正在绘制的所有数据。
尝试在 matrixPlotter 中的 return 语句之前放置一个 plt.close(fig)。如果 matplotlib (and I don't think it is).
未收集到它，那么您的 gc.collect() 语句应该能够捕获它
我会先描述脚本，然后发布代码和重现问题的方法，在我的机器上可能不是每台机器都是这样。
所以我有一个脚本可以构建具有特定属性的对象、云，并将数据导出到文件，该文件是名为 SHDOM 的软件的输入，该工具分析辐射属性并可在以下 link.
中进一步阅读剧本有点长，提前见谅：
import numpy as np
from itertools import product
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from mpl_toolkits.mplot3d import Axes3D
import gc
def writeDomainFile(type, file_name, temp_vec,
lwc, r_effective, dx, dy,
height_vec):
if type == 2:
assert list(lwc.shape) == list(r_effective.shape), "when the file type is 2 the shapes of lwc and r_effective shold be the same"
x_lwc, y_lwc, z_lwc = lwc.shape
x_r_effetive, y_r_effetive, z_r_effetive = r_effective.shape
fp = open(file_name, 'w')
# write the file format
fp.write("{:}\n".format(type))
# write the number of x, y, z points
fp.write("{:} {:} {:}\n".format(x_lwc, y_lwc, z_lwc))
# write the spacing resolution in x and y
fp.write("{:} {:}\n".format(dx, dy))
# write the height vector
fp.write(" ".join(map(str,height_vec)) + "\n")
# write the temprature vector
fp.write(" ".join(map(str,temp_vec)) + "\n")
indices = product(range(x_lwc), range(y_lwc), range(z_lwc))
print_indices = product(range(1, x_lwc + 1), range(1, y_lwc + 1), range(1, z_lwc + 1))
if type == 1: # only lwc
for triplet, print_triplets in zip(indices, print_indices):
# t = tuple(1 + list(triplet))
if not lwc[triplet]:
fp.write("{:} {:} {:} {:}\n".format(*print_triplets, 0))
else:
fp.write("{:} {:} {:} {:6.4f}\n".format(*print_triplets, lwc[triplet]))
elif type == 2:
for triplet, print_triplets in zip(indices, print_indices):
# t = tuple(map(lambda x: x + 1, list(triplet)))
if not lwc[triplet]:
fp.write("{:} {:} {:} {:} {:}\n".format(*print_triplets, 0, 0))
else:
fp.write("{:} {:} {:} {:6.4f} {:4.2f}\n".format(*print_triplets, lwc[triplet], r_effective[triplet]))
fp.close()
return
def lapseRateTemp(base_temp, position):
temp = np.zeros(position.shape)
temp[0] = base_temp
temp[1:] = base_temp - 6.5 * position[1:]
return np.round(temp, 4)
def createCubicCloudDomain(cloud_tick, x_size, y_size,
dx, dy, dz, r_effective_option, lwc_option,
cloud_x_size, cloud_y_size):
# const
temp0 = 300 # kelvin
N_x = x_size / dx # number of x points
N_y = y_size / dy # number of y points
cld_base = 1.0 # the cloud always start from 1[km] above ground
x_center = N_x / 2
y_center = N_y / 2
prog_x = (cloud_x_size / 2) / dx
prog_y = (cloud_y_size / 2) / dy
cloud_x_west = int(x_center - prog_x)
cloud_x_east = int(x_center + prog_x)
cloud_y_south = int(y_center - prog_y)
cloud_y_north = int(y_center + prog_y)
cloud_x_vec = np.arange(cloud_x_west, cloud_x_east, 1)
cloud_y_vec = np.arange(cloud_y_south, cloud_y_north, 1)
for tick in cloud_tick: # cloud_tick might be a vector
cld_top = cld_base + tick
N_z = tick / dz # number of z points
cloud_z_vec = np.round(np.arange(cld_base, cld_top, dz), 4)
# temprature
temp_base = temp0 - 9.8 * cld_base
temp_vec = lapseRateTemp(temp_base, cloud_z_vec - cld_base,)
temp_cloud = np.tile(temp_vec,(int(temp_vec.shape[0]), int(temp_vec.shape[0]), 1))
temp_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
temp_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = temp_cloud
plotTemperatureGradient(temp_domain, 'test.png')
# del temp_cloud
# del temp_domain
# gc.collect()
for r in r_effective_option:
for l in lwc_option:
r_effective_cloud = np.full((cloud_x_vec.shape[0],
cloud_y_vec.shape[0],
cloud_z_vec.shape[0]),
r)
lwc_cloud = np.full((cloud_x_vec.shape[0],
cloud_y_vec.shape[0],
cloud_z_vec.shape[0]),
l)
r_effective_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
lwc_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
# the positions to enter the cloud data
lwc_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = lwc_cloud
r_effective_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = r_effective_cloud
writeDomainFile(2, "test.txt", temp_vec, lwc_domain, r_effective_domain, dx, dy, cloud_z_vec)
plotGeneratedCloud(lwc_domain, r_effective_domain)
return
def plotTemperatureGradient(temp_mat, file_name):
xs, ys, zs = temp_mat.shape
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
X, Y, Z = np.mgrid[:xs, :ys, :zs]
faltten_data = temp_mat.ravel().astype(np.float16)
color_map = np.zeros((faltten_data.shape[0], 4))
# map scalars to colors
minima = np.min(faltten_data[np.nonzero(faltten_data)])
maxima = np.max(faltten_data[np.nonzero(faltten_data)])
norm = matplotlib.colors.Normalize(vmin=minima, vmax=maxima, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap='jet')
rgba = mapper.to_rgba(faltten_data)
color_map[:,0:3] = rgba[:, 0:3]
color_map[:,3] = np.where(faltten_data > 0, 0.07, 0)
p = ax.scatter(X, Y, Z, c=color_map.astype(np.float16))
ax.set_xlabel('X position [Arb.]')
ax.set_ylabel('Y position [Arb.]')
ax.set_zlabel('Z position [Arb.]')
fig.colorbar(p)
# plt.title(title)
plt.savefig(file_name)
return
def plotGeneratedCloud(lwc, r_effective):
light_blue = np.array([0, 1, 0.7])
matrixPlotter(lwc, 'lwc.png',
'Liquid water content of the cloud',
light_blue)
# gc.collect()
light_pink = np.array([0.9, 0.6, 0.9])
matrixPlotter(r_effective, 'r_effective.png',
'Effective radius of the cloud',
light_pink)
return
def matrixPlotter(mat, file_name, title, c=np.array([0.2, 0.6, 0])):
xs, ys, zs = mat.shape
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
X, Y, Z = np.mgrid[:xs, :ys, :zs]
faltten_data = mat.ravel().astype(np.float16)
color_map = np.zeros((faltten_data.shape[0], 4))
color_map[:,0:3] = c
color_map[:,3] = np.where(faltten_data > 0, 0.07, 0)
ax.scatter(X, Y, Z, c=color_map.astype(np.float16))
ax.set_xlabel('X position [Arb.]')
ax.set_ylabel('Y position [Arb.]')
ax.set_zlabel('Z position [Arb.]')
plt.title(title)
plt.savefig(file_name)
return
# def main():
# return
if __name__ == '__main__':
# main()
createCubicCloudDomain([0.5], 2.02, 2.02, 0.01, 0.01, 0.01, [0.5], [1], 0.5, 0.5)
print("Done")
所以问题发生在：
- 我尝试导出所有 3 个图，调用以下块：
plotGeneratedCloud(lwc_domain, r_effective_domain)或temp_cloud = np.tile(temp_vec,(int(temp_vec.shape[0]), int(temp_vec.shape[0]), 1)) temp_domain = np.zeros((int(N_x), int(N_y), int(N_z))) temp_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = temp_cloud plotTemperatureGradient(temp_domain, 'test.png') - 当我尝试创建超过 1 个对象（云）时，调用函数时：
createCubicCloudDomain([0.5, 0.6], 2.02, 2.02, 0.01, 0.01, 0.01, [0.5, 0.4], [1, 0.3], 0.5, 0.5) - 增加域大小，例如：
createCubicCloudDomain([0.5], 4.02, 4.02, 0.01, 0.01, 0.01, [0.5], [1], 0.5, 0.5)
我遇到的错误类型如下：
File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\backends\backend_agg.py", line 396, in draw self.figure.draw(self.renderer) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\artist.py", line 38, in draw_wrapper return draw(artist, renderer, *args, **kwargs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\figure.py", line 1735, in draw mimage._draw_list_compositing_images( File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\image.py", line 137, in _draw_list_compositing_images a.draw(renderer) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\artist.py", line 38, in draw_wrapper return draw(artist, renderer, *args, **kwargs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\axes3d.py", line 291, in draw sorted(self.collections, File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\axes3d.py", line 292, in key=lambda col: col.do_3d_projection(renderer), File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\art3d.py", line 538, in do_3d_projection vxs, vys, vzs, vis = proj3d.proj_transform_clip(xs, ys, zs, renderer.M) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\proj3d.py", line 214, in proj_transform_clip vec = _vec_pad_ones(xs, ys, zs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\proj3d.py", line 189, in _vec_pad_ones return np.array([xs, ys, zs, np.ones_like(xs)]) MemoryError: Unable to allocate array with shape (4, 2040200) and data type float64
我知道问题是内存问题，所以我认为一旦完成删除数组可能会解决问题，所以我使用了 del 函数，甚至使用了以下问题中使用的垃圾收集器解决方案: How can I explicitly free memory in Python?
希望得到一些帮助
如果您不先.close() figure 对象，它会保留在内存中，并且它包含它正在绘制的所有数据。
尝试在 matrixPlotter 中的 return 语句之前放置一个 plt.close(fig)。如果 matplotlib (and I don't think it is).
gc.collect() 语句应该能够捕获它