删除keras Sequential模型中数组的空白行和列

Question

我有一个 keras 模型，它采用 10x10x1 数组作为输入。例如：

array = np.array([
[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],
[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],
[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],
[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],
[[0],[0],[0],[0],[1],[1],[0],[0],[0],[0]],
[[0],[0],[0],[0],[1],[0],[0],[0],[0],[0]],
[[0],[0],[0],[0],[1],[1],[0],[0],[0],[0]],
[[0],[0],[0],[0],[0],[1],[0],[0],[0],[0]],
[[0],[0],[0],[0],[1],[1],[0],[0],[0],[0]],
[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]]
])

这个数组表示数字“5”

我想要做的是，删除所有 'blank'（像素值为 0）的行和列。
结果数组的形状为 5x2x1,

array = np.array([
[[1],[1]],
[[1],[0]],
[[1],[1]],
[[0],[1]],
[[1],[1]]
])

它看起来像这样：

我想在 keras 模型中做这个，作为一个层，所以它必须用 tensorflow.keras.backend 来完成。
它可以作为一个 Lambda 层，像这样：

def remove_blank(x):
   ...
   return x_

model.add(Lambda(remove_blank))

此外，此函数也应该能够对 RGB 图像执行此操作，这意味着形状为 (HEIGHT, WIDTH, 3) 的 3D 数组。

RGB 图像的示例为：

array = np.array([
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]]
], dtype='float')

该函数应删除仅包含黑色像素的行和列，并将此图像转换为：

array = np.array([
[[1,1,1],[1,1,1]],
[[1,1,1],[0,0,0]],
[[1,1,1],[1,1,1]],
[[0,0,0],[1,1,1]],
[[1,1,1],[1,1,1]]
], dtype='float')

Answer 1

您可以尝试这样的操作，但请注意，在 graph 模式下迭代张量将不起作用，因此请使用 tf.while_loop 来表示 batch_size > 1。如果 batch_size = 1，你不需要任何循环：

import tensorflow as tf
import matplotlib.pyplot as plt

five = tf.constant([
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]]
], dtype=tf.float32)

eight = tf.constant([
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [0], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [0], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]]
], dtype=tf.float32)

def remove_blank(t):
  ragged = []
  for i in tf.range(tf.shape(t)[0]):
    tensor = tf.expand_dims(t[i, :, :, 0], axis=0)
    x = tf.where(tf.not_equal(tensor, 0.0))
    _min = tf.reduce_min(x, axis=0)
    _max = tf.reduce_max(x, axis=0)
    col_range = tf.range(_min[1], _max[1] + 1)
    row_range = tf.range(_min[2], _max[2] + 1)

    i = tf.repeat(col_range, repeats=tf.shape(row_range)[0])
    j = tf.tile(row_range, multiples=[tf.shape(col_range)[0]])
    b = tf.zeros(tf.shape(i)[0], dtype=tf.int64)

    z = tf.gather_nd(tensor, tf.transpose(tf.stack([b, i, j])))
    z = tf.reshape(z, (tf.shape(tf.unique(x[:, 1])[0])[0], tf.shape(tf.unique(x[:, 2])[0])[0], tf.shape(t)[-1]))
    ragged.append(z)
  return tf.ragged.stack(ragged)

x = tf.stack([five, eight])

model = tf.keras.Sequential()
model.add(tf.keras.layers.Lambda(remove_blank))
output = model(x)

f = plt.figure(figsize=(10,3))
ax1 = f.add_subplot(121)
ax2 = f.add_subplot(122)
ax1.imshow(five[:, :, 0], cmap='gray')
ax2.imshow(eight[:, :, 0], cmap='gray')

f1 = plt.figure(figsize=(10,3))
ax1 = f1.add_subplot(121)
ax2 = f1.add_subplot(122)
ax1.imshow(output[0].to_tensor()[:, :, 0], cmap='gray')
ax2.imshow(output[1].to_tensor()[:, :, 0], cmap='gray')

训练模型时，您必须将图像调整为统一大小。我将留给您解决如何为 RGB 图像扩展此方法 ;)

更新 1：

下面是集成到 Keras 模型中的方法示例：

import tensorflow as tf
import matplotlib.pyplot as plt

five = tf.constant([
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]]
], dtype=tf.float32)

eight = tf.constant([
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [0], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [0], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [1], [1], [1], [0], [0], [0], [0]],
    [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]]
], dtype=tf.float32)

def remove_blank(t):
  def _remove_blank(i0, v, t):
    tensor = tf.expand_dims(t[i0, :, :, 0], axis=0)
    x = tf.where(tf.not_equal(tensor, 0.0))
    _min = tf.reduce_min(x, axis=0)
    _max = tf.reduce_max(x, axis=0)
    col_range = tf.range(_min[1], _max[1] + 1)
    row_range = tf.range(_min[2], _max[2] + 1)

    i = tf.repeat(col_range, repeats=tf.shape(row_range)[0])
    j = tf.tile(row_range, multiples=[tf.shape(col_range)[0]])
    b = tf.zeros(tf.shape(i)[0], dtype=tf.int64)

    z = tf.gather_nd(tensor, tf.transpose(tf.stack([b, i, j])))
    z = tf.reshape(z, (tf.shape(tf.unique(x[:, 1])[0])[0], tf.shape(tf.unique(x[:, 2])[0])[0], tf.shape(t)[-1]))
    z = tf.expand_dims(tf.image.resize(z, [5, 2]), axis=0)

    h = tf.range(tf.shape(z)[1], dtype=tf.int32)
    ij = tf.expand_dims(tf.stack([tf.repeat([i0], repeats=tf.shape(h)[0]), h], axis=-1), axis=0)
    v = tf.tensor_scatter_nd_update(v, ij, z)

    return tf.add(i0, 1), v, t

  i0 = tf.constant(0, dtype=tf.int32)
  v = tf.zeros((tf.shape(t)[0], 5, 2, 1))
  while_condition = lambda i0, v, t: tf.less(i0, tf.shape(t)[0])
  _, v, _ = tf.while_loop(while_condition, _remove_blank, loop_vars=(i0, v, t))
  return v

batch_size = 2
inputs = tf.keras.layers.Input((10, 10, 1))
outputs = tf.keras.layers.Lambda(remove_blank)(inputs)
model = tf.keras.Model(inputs, outputs)

y = tf.random.normal((2, 5, 2, 1))
model.compile(optimizer='adam', loss='mse')
model.fit(x, y, batch_size=2, epochs=4)

Epoch 1/4
1/1 [==============================] - 1s 860ms/step - loss: 1.1193
Epoch 2/4
1/1 [==============================] - 0s 17ms/step - loss: 1.1193
Epoch 3/4
1/1 [==============================] - 0s 16ms/step - loss: 1.1193
Epoch 4/4
1/1 [==============================] - 0s 16ms/step - loss: 1.1193
<keras.callbacks.History at 0x7f11b0350510>

RGB 图像的解决方案如下所示：

import tensorflow as tf
import matplotlib.pyplot as plt

five = tf.constant([
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[0,0,0],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[1,1,1],[1,1,1],[0,0,0]],
[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]]
], dtype=tf.float32)

def _remove_blank(i, t, channel):
    tensor = tf.expand_dims(t[i, :, :, channel], axis=0)
    x = tf.where(tf.not_equal(tensor, 0.0))
    _min = tf.reduce_min(x, axis=0)
    _max = tf.reduce_max(x, axis=0)
    col_range = tf.range(_min[1], _max[1] + 1)
    row_range = tf.range(_min[2], _max[2] + 1)

    i = tf.repeat(col_range, repeats=tf.shape(row_range)[0])
    j = tf.tile(row_range, multiples=[tf.shape(col_range)[0]])
    b = tf.zeros(tf.shape(i)[0], dtype=tf.int64)

    z = tf.gather_nd(tensor, tf.transpose(tf.stack([b, i, j])))
    z = tf.reshape(z, (tf.shape(tf.unique(x[:, 1])[0])[0], tf.shape(tf.unique(x[:, 2])[0])[0], 1))
    return z

def remove_blank(t):
  ragged = []
  for i in tf.range(tf.shape(t)[0]): 
    z = tf.concat([_remove_blank(i, t, channel=0), _remove_blank(i, t, channel=1), _remove_blank(i, t, channel=2)], axis = -1)
    ragged.append(z)
  return tf.ragged.stack(ragged)

x = tf.expand_dims(five, axis=0)

model = tf.keras.Sequential()
model.add(tf.keras.layers.Lambda(remove_blank))
output = model(x)

f = plt.figure(figsize=(10,3))
ax1 = f.add_subplot(121)
ax1.imshow(five)

f1 = plt.figure(figsize=(10,3))
ax1 = f1.add_subplot(121)
ax1.imshow(output[0].to_tensor())

删除keras Sequential模型中数组的空白行和列

Remove blank rows and columns of an array inside a keras Sequential model

python

machine-learning

keras

tensorflow

tensorflow2.0

更新 1：