如何在 Tensorboard 中显示自定义图像（例如 Matplotlib Plots）？
How to Display Custom Images in Tensorboard (e.g. Matplotlib Plots)?
Tensorboard 自述文件的 Image Dashboard 部分说：
Since the image dashboard supports arbitrary pngs, you can use this to embed custom visualizations (e.g. matplotlib scatterplots) into TensorBoard.
我知道如何将 pyplot 图像写入文件，作为张量读回，然后与 tf.image_summary() 一起使用以将其写入 TensorBoard，但自述文件中的这一声明表明存在更直接的方式。在那儿？如果是这样，是否有任何进一步的文档 and/or 示例来说明如何有效地执行此操作？
如果你有内存缓冲区中的图像，这很容易做到。下面，我展示了一个示例，其中将 pyplot 保存到缓冲区，然后转换为 TF 图像表示，然后将其发送到图像摘要。
import io
import matplotlib.pyplot as plt
import tensorflow as tf
def gen_plot():
"""Create a pyplot plot and save to buffer."""
plt.figure()
plt.plot([1, 2])
plt.title("test")
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
return buf
# Prepare the plot
plot_buf = gen_plot()
# Convert PNG buffer to TF image
image = tf.image.decode_png(plot_buf.getvalue(), channels=4)
# Add the batch dimension
image = tf.expand_dims(image, 0)
# Add image summary
summary_op = tf.summary.image("plot", image)
# Session
with tf.Session() as sess:
# Run
summary = sess.run(summary_op)
# Write summary
writer = tf.train.SummaryWriter('./logs')
writer.add_summary(summary)
writer.close()
这给出了以下 TensorBoard 可视化：
下一个脚本不使用中间 RGB/PNG 编码。它还修复了执行期间额外操作构造的问题，单个摘要被重用。
图形的大小预计在执行期间保持不变
有效的解决方案：
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
def get_figure():
fig = plt.figure(num=0, figsize=(6, 4), dpi=300)
fig.clf()
return fig
def fig2rgb_array(fig, expand=True):
fig.canvas.draw()
buf = fig.canvas.tostring_rgb()
ncols, nrows = fig.canvas.get_width_height()
shape = (nrows, ncols, 3) if not expand else (1, nrows, ncols, 3)
return np.fromstring(buf, dtype=np.uint8).reshape(shape)
def figure_to_summary(fig):
image = fig2rgb_array(fig)
summary_writer.add_summary(
vis_summary.eval(feed_dict={vis_placeholder: image}))
if __name__ == '__main__':
# construct graph
x = tf.Variable(initial_value=tf.random_uniform((2, 10)))
inc = x.assign(x + 1)
# construct summary
fig = get_figure()
vis_placeholder = tf.placeholder(tf.uint8, fig2rgb_array(fig).shape)
vis_summary = tf.summary.image('custom', vis_placeholder)
with tf.Session() as sess:
tf.global_variables_initializer().run()
summary_writer = tf.summary.FileWriter('./tmp', sess.graph)
for i in range(100):
# execute step
_, values = sess.run([inc, x])
# draw on the plot
fig = get_figure()
plt.subplot('111').scatter(values[0], values[1])
# save the summary
figure_to_summary(fig)
这是为了完成 Andrzej Pronobis 的回答。密切关注他的好 post，我设置了这个 最小工作示例 :
plt.figure()
plt.plot([1, 2])
plt.title("test")
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
image = tf.image.decode_png(buf.getvalue(), channels=4)
image = tf.expand_dims(image, 0)
summary = tf.summary.image("test", image, max_outputs=1)
writer.add_summary(summary, step)
writer 是 tf.summary.FileWriter 的实例。
这给了我以下错误：
AttributeError: 'Tensor' 对象没有属性 'value'
this github post 有解决方案：在将摘要添加到编写器之前必须对其进行评估（转换为字符串）。所以我的工作代码仍然如下（只需在最后一行添加 .eval() 调用）：
plt.figure()
plt.plot([1, 2])
plt.title("test")
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
image = tf.image.decode_png(buf.getvalue(), channels=4)
image = tf.expand_dims(image, 0)
summary = tf.summary.image("test", image, max_outputs=1)
writer.add_summary(summary.eval(), step)
这可能很短，可以作为对他的回答的评论，但这些很容易被忽略（而且我可能也在做其他不同的事情），所以在这里，希望它有所帮助！
我的回答有点晚了。 tf-matplotlib 一个简单的散点图归结为：
import tensorflow as tf
import numpy as np
import tfmpl
@tfmpl.figure_tensor
def draw_scatter(scaled, colors):
'''Draw scatter plots. One for each color.'''
figs = tfmpl.create_figures(len(colors), figsize=(4,4))
for idx, f in enumerate(figs):
ax = f.add_subplot(111)
ax.axis('off')
ax.scatter(scaled[:, 0], scaled[:, 1], c=colors[idx])
f.tight_layout()
return figs
with tf.Session(graph=tf.Graph()) as sess:
# A point cloud that can be scaled by the user
points = tf.constant(
np.random.normal(loc=0.0, scale=1.0, size=(100, 2)).astype(np.float32)
)
scale = tf.placeholder(tf.float32)
scaled = points*scale
# Note, `scaled` above is a tensor. Its being passed `draw_scatter` below.
# However, when `draw_scatter` is invoked, the tensor will be evaluated and a
# numpy array representing its content is provided.
image_tensor = draw_scatter(scaled, ['r', 'g'])
image_summary = tf.summary.image('scatter', image_tensor)
all_summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter('log', sess.graph)
summary = sess.run(all_summaries, feed_dict={scale: 2.})
writer.add_summary(summary, global_step=0)
执行后，在 Tensorboard 中生成以下图
请注意，tf-matplotlib 负责评估任何张量输入，避免 pyplot 线程问题，并支持运行时关键绘图的 blitting。
最后 official documentation 关于“记录任意图像数据”的内容，其中包含 matplotlib 创建的图像示例。
他们写道：
在下面的代码中，您将使用 matplotlib 的 subplot() 函数将前 25 张图像记录为一个漂亮的网格。然后您将在 TensorBoard 中查看网格：
# Clear out prior logging data.
!rm -rf logs/plots
logdir = "logs/plots/" + datetime.now().strftime("%Y%m%d-%H%M%S")
file_writer = tf.summary.create_file_writer(logdir)
def plot_to_image(figure):
"""Converts the matplotlib plot specified by 'figure' to a PNG image and
returns it. The supplied figure is closed and inaccessible after this call."""
# Save the plot to a PNG in memory.
buf = io.BytesIO()
plt.savefig(buf, format='png')
# Closing the figure prevents it from being displayed directly inside
# the notebook.
plt.close(figure)
buf.seek(0)
# Convert PNG buffer to TF image
image = tf.image.decode_png(buf.getvalue(), channels=4)
# Add the batch dimension
image = tf.expand_dims(image, 0)
return image
def image_grid():
"""Return a 5x5 grid of the MNIST images as a matplotlib figure."""
# Create a figure to contain the plot.
figure = plt.figure(figsize=(10,10))
for i in range(25):
# Start next subplot.
plt.subplot(5, 5, i + 1, title=class_names[train_labels[i]])
plt.xticks([])
plt.yticks([])
plt.grid(False)
plt.imshow(train_images[i], cmap=plt.cm.binary)
return figure
# Prepare the plot
figure = image_grid()
# Convert to image and log
with file_writer.as_default():
tf.summary.image("Training data", plot_to_image(figure), step=0)
%tensorboard --logdir logs/plots
已弃用：对于 PyTorch，请使用内置的 SummaryWriter.add_figure (see other )!
PyTorch 解决方案：
- 使用 MatPlotLib 图
- 将其绘制到 canvas
- 然后转换为 numpy:
# make the canvas
figure = plt.figure(figsize=(10,10))
canvas = matplotlib.backends.backend_agg.FigureCanvas(figure)
# insert plotting code here; you can use imshow or subplot, etc.
for i in range(25):
plt.subplot(5, 5, i + 1, title=class_names[train_labels[i]])
plt.xticks([])
plt.yticks([])
plt.grid(False)
plt.imshow(train_images[i], cmap=plt.cm.binary)
# convert canvas to figure
canvas.draw()
image = np.frombuffer(canvas.tostring_rgb(), dtype='uint8').reshape((1000,1000,3)).transpose((2, 0, 1))
结果可以直接添加到Tensorboard：
tensorboard.add_image('name', image, global_step)
Pytorch Lightning中的一个解决方案
这不是完整的 class，而是您必须添加才能使其在框架中运行的内容。
import pytorch_lightning as pl
import seaborn as sn
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
def __init__(self, config, trained_vae, latent_dim):
self.val_confusion = pl.metrics.classification.ConfusionMatrix(num_classes=self._config.n_clusters)
self.logger: Optional[TensorBoardLogger] = None
def forward(self, x):
...
return log_probs
def validation_step(self, batch, batch_index):
if self._config.dataset == "mnist":
orig_batch, label_batch = batch
orig_batch = orig_batch.reshape(-1, 28 * 28)
log_probs = self.forward(orig_batch)
loss = self._criterion(log_probs, label_batch)
self.val_confusion.update(log_probs, label_batch)
return {"loss": loss, "labels": label_batch}
def validation_step_end(self, outputs):
return outputs
def validation_epoch_end(self, outs):
tb = self.logger.experiment
# confusion matrix
conf_mat = self.val_confusion.compute().detach().cpu().numpy().astype(np.int)
df_cm = pd.DataFrame(
conf_mat,
index=np.arange(self._config.n_clusters),
columns=np.arange(self._config.n_clusters))
plt.figure()
sn.set(font_scale=1.2)
sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, fmt='d')
buf = io.BytesIO()
plt.savefig(buf, format='jpeg')
buf.seek(0)
im = Image.open(buf)
im = torchvision.transforms.ToTensor()(im)
tb.add_image("val_confusion_matrix", im, global_step=self.current_epoch)
和通话
logger = TensorBoardLogger(save_dir=tb_logs_folder, name='Classifier')
trainer = Trainer(
default_root_dir=classifier_checkpoints_path,
logger=logger,
)
可以使用 add_figure 函数直接将 Matplotlib 图添加到张量板：
import numpy as np, matplotlib.pyplot as plt
from torch.utils.tensorboard import SummaryWriter
# Example plot
x = np.linspace(0,10)
plt.plot(x, np.sin(x))
# Adding plot to tensorboard
with SummaryWriter('runs/SO_test') as writer:
writer.add_figure('Fig1', plt.gcf())
# Loading tensorboard
%tensorboard --logdir=runs
Tensorboard 自述文件的 Image Dashboard 部分说：
Since the image dashboard supports arbitrary pngs, you can use this to embed custom visualizations (e.g. matplotlib scatterplots) into TensorBoard.
我知道如何将 pyplot 图像写入文件，作为张量读回，然后与 tf.image_summary() 一起使用以将其写入 TensorBoard，但自述文件中的这一声明表明存在更直接的方式。在那儿？如果是这样，是否有任何进一步的文档 and/or 示例来说明如何有效地执行此操作？
如果你有内存缓冲区中的图像，这很容易做到。下面，我展示了一个示例，其中将 pyplot 保存到缓冲区，然后转换为 TF 图像表示，然后将其发送到图像摘要。
import io
import matplotlib.pyplot as plt
import tensorflow as tf
def gen_plot():
"""Create a pyplot plot and save to buffer."""
plt.figure()
plt.plot([1, 2])
plt.title("test")
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
return buf
# Prepare the plot
plot_buf = gen_plot()
# Convert PNG buffer to TF image
image = tf.image.decode_png(plot_buf.getvalue(), channels=4)
# Add the batch dimension
image = tf.expand_dims(image, 0)
# Add image summary
summary_op = tf.summary.image("plot", image)
# Session
with tf.Session() as sess:
# Run
summary = sess.run(summary_op)
# Write summary
writer = tf.train.SummaryWriter('./logs')
writer.add_summary(summary)
writer.close()
这给出了以下 TensorBoard 可视化：
下一个脚本不使用中间 RGB/PNG 编码。它还修复了执行期间额外操作构造的问题，单个摘要被重用。
图形的大小预计在执行期间保持不变
有效的解决方案：
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
def get_figure():
fig = plt.figure(num=0, figsize=(6, 4), dpi=300)
fig.clf()
return fig
def fig2rgb_array(fig, expand=True):
fig.canvas.draw()
buf = fig.canvas.tostring_rgb()
ncols, nrows = fig.canvas.get_width_height()
shape = (nrows, ncols, 3) if not expand else (1, nrows, ncols, 3)
return np.fromstring(buf, dtype=np.uint8).reshape(shape)
def figure_to_summary(fig):
image = fig2rgb_array(fig)
summary_writer.add_summary(
vis_summary.eval(feed_dict={vis_placeholder: image}))
if __name__ == '__main__':
# construct graph
x = tf.Variable(initial_value=tf.random_uniform((2, 10)))
inc = x.assign(x + 1)
# construct summary
fig = get_figure()
vis_placeholder = tf.placeholder(tf.uint8, fig2rgb_array(fig).shape)
vis_summary = tf.summary.image('custom', vis_placeholder)
with tf.Session() as sess:
tf.global_variables_initializer().run()
summary_writer = tf.summary.FileWriter('./tmp', sess.graph)
for i in range(100):
# execute step
_, values = sess.run([inc, x])
# draw on the plot
fig = get_figure()
plt.subplot('111').scatter(values[0], values[1])
# save the summary
figure_to_summary(fig)
这是为了完成 Andrzej Pronobis 的回答。密切关注他的好 post，我设置了这个 最小工作示例 :
plt.figure()
plt.plot([1, 2])
plt.title("test")
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
image = tf.image.decode_png(buf.getvalue(), channels=4)
image = tf.expand_dims(image, 0)
summary = tf.summary.image("test", image, max_outputs=1)
writer.add_summary(summary, step)
writer 是 tf.summary.FileWriter 的实例。 这给了我以下错误： AttributeError: 'Tensor' 对象没有属性 'value' this github post 有解决方案：在将摘要添加到编写器之前必须对其进行评估（转换为字符串）。所以我的工作代码仍然如下（只需在最后一行添加 .eval() 调用）：
plt.figure()
plt.plot([1, 2])
plt.title("test")
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
image = tf.image.decode_png(buf.getvalue(), channels=4)
image = tf.expand_dims(image, 0)
summary = tf.summary.image("test", image, max_outputs=1)
writer.add_summary(summary.eval(), step)
这可能很短，可以作为对他的回答的评论，但这些很容易被忽略（而且我可能也在做其他不同的事情），所以在这里，希望它有所帮助！
我的回答有点晚了。 tf-matplotlib 一个简单的散点图归结为：
import tensorflow as tf
import numpy as np
import tfmpl
@tfmpl.figure_tensor
def draw_scatter(scaled, colors):
'''Draw scatter plots. One for each color.'''
figs = tfmpl.create_figures(len(colors), figsize=(4,4))
for idx, f in enumerate(figs):
ax = f.add_subplot(111)
ax.axis('off')
ax.scatter(scaled[:, 0], scaled[:, 1], c=colors[idx])
f.tight_layout()
return figs
with tf.Session(graph=tf.Graph()) as sess:
# A point cloud that can be scaled by the user
points = tf.constant(
np.random.normal(loc=0.0, scale=1.0, size=(100, 2)).astype(np.float32)
)
scale = tf.placeholder(tf.float32)
scaled = points*scale
# Note, `scaled` above is a tensor. Its being passed `draw_scatter` below.
# However, when `draw_scatter` is invoked, the tensor will be evaluated and a
# numpy array representing its content is provided.
image_tensor = draw_scatter(scaled, ['r', 'g'])
image_summary = tf.summary.image('scatter', image_tensor)
all_summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter('log', sess.graph)
summary = sess.run(all_summaries, feed_dict={scale: 2.})
writer.add_summary(summary, global_step=0)
执行后，在 Tensorboard 中生成以下图
请注意，tf-matplotlib 负责评估任何张量输入，避免 pyplot 线程问题，并支持运行时关键绘图的 blitting。
最后 official documentation 关于“记录任意图像数据”的内容，其中包含 matplotlib 创建的图像示例。
他们写道：
在下面的代码中，您将使用 matplotlib 的 subplot() 函数将前 25 张图像记录为一个漂亮的网格。然后您将在 TensorBoard 中查看网格：
# Clear out prior logging data.
!rm -rf logs/plots
logdir = "logs/plots/" + datetime.now().strftime("%Y%m%d-%H%M%S")
file_writer = tf.summary.create_file_writer(logdir)
def plot_to_image(figure):
"""Converts the matplotlib plot specified by 'figure' to a PNG image and
returns it. The supplied figure is closed and inaccessible after this call."""
# Save the plot to a PNG in memory.
buf = io.BytesIO()
plt.savefig(buf, format='png')
# Closing the figure prevents it from being displayed directly inside
# the notebook.
plt.close(figure)
buf.seek(0)
# Convert PNG buffer to TF image
image = tf.image.decode_png(buf.getvalue(), channels=4)
# Add the batch dimension
image = tf.expand_dims(image, 0)
return image
def image_grid():
"""Return a 5x5 grid of the MNIST images as a matplotlib figure."""
# Create a figure to contain the plot.
figure = plt.figure(figsize=(10,10))
for i in range(25):
# Start next subplot.
plt.subplot(5, 5, i + 1, title=class_names[train_labels[i]])
plt.xticks([])
plt.yticks([])
plt.grid(False)
plt.imshow(train_images[i], cmap=plt.cm.binary)
return figure
# Prepare the plot
figure = image_grid()
# Convert to image and log
with file_writer.as_default():
tf.summary.image("Training data", plot_to_image(figure), step=0)
%tensorboard --logdir logs/plots
已弃用：对于 PyTorch，请使用内置的 SummaryWriter.add_figure (see other
PyTorch 解决方案：
- 使用 MatPlotLib 图
- 将其绘制到 canvas
- 然后转换为 numpy:
# make the canvas
figure = plt.figure(figsize=(10,10))
canvas = matplotlib.backends.backend_agg.FigureCanvas(figure)
# insert plotting code here; you can use imshow or subplot, etc.
for i in range(25):
plt.subplot(5, 5, i + 1, title=class_names[train_labels[i]])
plt.xticks([])
plt.yticks([])
plt.grid(False)
plt.imshow(train_images[i], cmap=plt.cm.binary)
# convert canvas to figure
canvas.draw()
image = np.frombuffer(canvas.tostring_rgb(), dtype='uint8').reshape((1000,1000,3)).transpose((2, 0, 1))
结果可以直接添加到Tensorboard：
tensorboard.add_image('name', image, global_step)
Pytorch Lightning中的一个解决方案
这不是完整的 class，而是您必须添加才能使其在框架中运行的内容。
import pytorch_lightning as pl
import seaborn as sn
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
def __init__(self, config, trained_vae, latent_dim):
self.val_confusion = pl.metrics.classification.ConfusionMatrix(num_classes=self._config.n_clusters)
self.logger: Optional[TensorBoardLogger] = None
def forward(self, x):
...
return log_probs
def validation_step(self, batch, batch_index):
if self._config.dataset == "mnist":
orig_batch, label_batch = batch
orig_batch = orig_batch.reshape(-1, 28 * 28)
log_probs = self.forward(orig_batch)
loss = self._criterion(log_probs, label_batch)
self.val_confusion.update(log_probs, label_batch)
return {"loss": loss, "labels": label_batch}
def validation_step_end(self, outputs):
return outputs
def validation_epoch_end(self, outs):
tb = self.logger.experiment
# confusion matrix
conf_mat = self.val_confusion.compute().detach().cpu().numpy().astype(np.int)
df_cm = pd.DataFrame(
conf_mat,
index=np.arange(self._config.n_clusters),
columns=np.arange(self._config.n_clusters))
plt.figure()
sn.set(font_scale=1.2)
sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, fmt='d')
buf = io.BytesIO()
plt.savefig(buf, format='jpeg')
buf.seek(0)
im = Image.open(buf)
im = torchvision.transforms.ToTensor()(im)
tb.add_image("val_confusion_matrix", im, global_step=self.current_epoch)
和通话
logger = TensorBoardLogger(save_dir=tb_logs_folder, name='Classifier')
trainer = Trainer(
default_root_dir=classifier_checkpoints_path,
logger=logger,
)
可以使用 add_figure 函数直接将 Matplotlib 图添加到张量板：
import numpy as np, matplotlib.pyplot as plt
from torch.utils.tensorboard import SummaryWriter
# Example plot
x = np.linspace(0,10)
plt.plot(x, np.sin(x))
# Adding plot to tensorboard
with SummaryWriter('runs/SO_test') as writer:
writer.add_figure('Fig1', plt.gcf())
# Loading tensorboard
%tensorboard --logdir=runs