model.fit 在 keras.sequential 上使用 tf.data.Dataset 时引发 ValueError
model.fit on keras.sequential when using tf.data.Dataset raises a ValueError
我正在尝试使用 tf.data.dataset 作为 Keras.sequential 的输入在 tensorflow 1.10 上构建我的第一个分类器，但是 fit 方法 returns 出现以下错误：
ValueError: Error when checking target: expected dense_1 to have 2 dimensions, but got array with shape (None,)
首先我用数据集的文件名初始化了 2 tf.data.Dataset
#Initialize dataset directories location and parameters
image_size=50
batch_size=10
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'
#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)
然后我用map方法准备了我的数据集
#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))
def load_resize_label(filename, label):
image_string = tf.read_file(filename)
image_decoded = tf.image.decode_jpeg(image_string)
image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
image_resized=image_resized/255.0
return image_resized, tf.convert_to_tensor(label)
然后，我将数据集连接成一个最终数据集并初始化批量大小
#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
最后使用模型对象的compile和fit方法
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(dataset, epochs=10, steps_per_epoch=30)
（完整代码如下）
我正在使用：
Windows 10 64 位
cudnn-9.0-windows10-x64-v7.2.1.38
cuda_9.0.176_win10
tensorflow-GPU 1.10.0
import tensorflow as tf
from tensorflow import keras
image_size=50
batch_size=10
# Reads an image from a file, decodes it into a dense tensor, resizes it
# to a fixed shape.
def load_resize_label(filename, label):
image_string = tf.read_file(filename)
image_decoded = tf.image.decode_jpeg(image_string)
image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
image_resized=image_resized/255.0
return image_resized, tf.convert_to_tensor(label)
#Initialize dataset directories location
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'
#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)
#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset = mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset = ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))
#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
#the CNN architecture
model = keras.Sequential([
keras.layers.Convolution2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(image_size, image_size,3)),
keras.layers.MaxPool2D(pool_size=2),
keras.layers.BatchNormalization(),
keras.layers.Flatten(),
keras.layers.Dense(128, activation=tf.nn.relu),
keras.layers.Dropout(0.3),
keras.layers.Dense(2, activation=tf.nn.softmax)
])
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(dataset, epochs=10, steps_per_epoch=30)
回溯：
Traceback (most recent call last):
File "C:/Users/Jonas/PycharmProjects/learning/lesson2.py", line 47, in <module>
model.fit(dataset, epochs=10, steps_per_epoch=30)
File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1278, in fit
validation_split=validation_split)
File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 917, in _standardize_user_data
exception_prefix='target')
File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training_utils.py", line 182, in standardize_input_data
'with shape ' + str(data_shape))
ValueError: Error when checking target: expected dense_1 to have 2 dimensions, but got array with shape (None,)
您的代码中缺少一些“=”。
每个数据集操作应该像：
dataset = dataset.some_ops(...)
您的代码应如下所示：
import tensorflow as tf
from tensorflow import keras
image_size=50
batch_size=10
# Reads an image from a file, decodes it into a dense tensor, resizes it
# to a fixed shape.
def load_resize_label(filename, label):
image_string = tf.read_file(filename)
image_decoded = tf.image.decode_jpeg(image_string)
image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
image_resized=image_resized/255.0
if label == 'morty':
label = [0, 1]
elif label == 'rick':
label = [1, 0]
else:
raise ValueError(label)
return image_resized, tf.convert_to_tensor(label)
#Initialize dataset directories location
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'
#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)
#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset = mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset = ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))
#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
#the CNN architecture
model = keras.Sequential([
keras.layers.Convolution2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(image_size, image_size, 3)),
keras.layers.MaxPool2D(pool_size=2),
keras.layers.BatchNormalization(),
keras.layers.Flatten(),
keras.layers.Dense(128, activation=tf.nn.relu),
keras.layers.Dropout(0.3),
keras.layers.Dense(2, activation=tf.nn.softmax)
])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(dataset, epochs=10, steps_per_epoch=30)
此外，我建议您使用 dataset.prefetch(None) 并在 map 函数中使用 num_parallel_calls 参数。 Here is why。 TLDR：速度更快。
我正在尝试使用 tf.data.dataset 作为 Keras.sequential 的输入在 tensorflow 1.10 上构建我的第一个分类器，但是 fit 方法 returns 出现以下错误：
ValueError: Error when checking target: expected dense_1 to have 2 dimensions, but got array with shape (None,)
首先我用数据集的文件名初始化了 2 tf.data.Dataset
#Initialize dataset directories location and parameters
image_size=50
batch_size=10
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'
#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)
然后我用map方法准备了我的数据集
#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))
def load_resize_label(filename, label):
image_string = tf.read_file(filename)
image_decoded = tf.image.decode_jpeg(image_string)
image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
image_resized=image_resized/255.0
return image_resized, tf.convert_to_tensor(label)
然后，我将数据集连接成一个最终数据集并初始化批量大小
#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
最后使用模型对象的compile和fit方法
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(dataset, epochs=10, steps_per_epoch=30)
（完整代码如下）
我正在使用：
Windows 10 64 位
cudnn-9.0-windows10-x64-v7.2.1.38
cuda_9.0.176_win10
tensorflow-GPU 1.10.0
import tensorflow as tf
from tensorflow import keras
image_size=50
batch_size=10
# Reads an image from a file, decodes it into a dense tensor, resizes it
# to a fixed shape.
def load_resize_label(filename, label):
image_string = tf.read_file(filename)
image_decoded = tf.image.decode_jpeg(image_string)
image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
image_resized=image_resized/255.0
return image_resized, tf.convert_to_tensor(label)
#Initialize dataset directories location
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'
#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)
#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset = mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset = ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))
#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
#the CNN architecture
model = keras.Sequential([
keras.layers.Convolution2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(image_size, image_size,3)),
keras.layers.MaxPool2D(pool_size=2),
keras.layers.BatchNormalization(),
keras.layers.Flatten(),
keras.layers.Dense(128, activation=tf.nn.relu),
keras.layers.Dropout(0.3),
keras.layers.Dense(2, activation=tf.nn.softmax)
])
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(dataset, epochs=10, steps_per_epoch=30)
回溯：
Traceback (most recent call last):
File "C:/Users/Jonas/PycharmProjects/learning/lesson2.py", line 47, in <module>
model.fit(dataset, epochs=10, steps_per_epoch=30)
File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1278, in fit
validation_split=validation_split)
File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 917, in _standardize_user_data
exception_prefix='target')
File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training_utils.py", line 182, in standardize_input_data
'with shape ' + str(data_shape))
ValueError: Error when checking target: expected dense_1 to have 2 dimensions, but got array with shape (None,)
您的代码中缺少一些“=”。
每个数据集操作应该像：
dataset = dataset.some_ops(...)
您的代码应如下所示：
import tensorflow as tf
from tensorflow import keras
image_size=50
batch_size=10
# Reads an image from a file, decodes it into a dense tensor, resizes it
# to a fixed shape.
def load_resize_label(filename, label):
image_string = tf.read_file(filename)
image_decoded = tf.image.decode_jpeg(image_string)
image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
image_resized=image_resized/255.0
if label == 'morty':
label = [0, 1]
elif label == 'rick':
label = [1, 0]
else:
raise ValueError(label)
return image_resized, tf.convert_to_tensor(label)
#Initialize dataset directories location
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'
#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)
#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset = mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset = ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))
#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
#the CNN architecture
model = keras.Sequential([
keras.layers.Convolution2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(image_size, image_size, 3)),
keras.layers.MaxPool2D(pool_size=2),
keras.layers.BatchNormalization(),
keras.layers.Flatten(),
keras.layers.Dense(128, activation=tf.nn.relu),
keras.layers.Dropout(0.3),
keras.layers.Dense(2, activation=tf.nn.softmax)
])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(dataset, epochs=10, steps_per_epoch=30)
此外，我建议您使用 dataset.prefetch(None) 并在 map 函数中使用 num_parallel_calls 参数。 Here is why。 TLDR：速度更快。