恢复 Keras 模型的训练
Resuming Training of Keras Model
我正在使用 tensorflow.keras 来构建一个具有 3 个密集层的简单神经网络。我能够成功训练模型 9000 个时期，达到 0.0496 的均方误差 (MSE)。不管怎样恢复模型，它在大约 57 MSE 时开始训练。
这可能表明模型权重未成功加载，但是当从头开始重新启动训练过程时（不加载以前保存的权重），MSE 从大约 +9000 开始。
编辑：
- 这是正常问题，还是我做错了什么？
- 为什么即使经过 9000 个迭代后准确率始终为 0.0？
我的代码如下：
from __future__ import absolute_import, division, print_function
import pathlib
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import model_from_json
from tensorflow.keras.models import load_model
print(tf.__version__)
dataset_path = 'D:\data.csv'
checkpoint_model_json_path = 'modelBackup/model.json'
checkpoint_weights_h5_path = 'modelBackup/weights00009000.h5'
resume_from_checkpoint = True
print('reading dataset...')
column_names = ['paircode','x1o','x1h','x1l','x1c','x1v','x2o','x2h','x2l','x2c','x2v','x3o','x3h','x3l','x3c','x3v','x4o','x4h','x4l','x4c','x4v','x5o','x5h','x5l','x5c','x5v','x6o','x6h','x6l','x6c','x6v','x7o','x7h','x7l','x7c','x7v','x8o','x8h','x8l','x8c','x8v','x9o','x9h','x9l','x9c','x9v','x10o','x10h','x10l','x10c','x10v','x11o','x11h','x11l','x11c','x11v','x12o','x12h','x12l','x12c','x12v','x13o','x13h','x13l','x13c','x13v','x14o','x14h','x14l','x14c','x14v','x15o','x15h','x15l','x15c','x15v','x16o','x16h','x16l','x16c','x16v','x17o','x17h','x17l','x17c','x17v','x18o','x18h','x18l','x18c','x18v','x19o','x19h','x19l','x19c','x19v','x20o','x20h','x20l','x20c','x20v','x21o','x21h','x21l','x21c','x21v','x22o','x22h','x22l','x22c','x22v','x23o','x23h','x23l','x23c','x23v','x24o','x24h','x24l','x24c','x24v','x25o','x25h','x25l','x25c','x25v','x26o','x26h','x26l','x26c','x26v','x27o','x27h','x27l','x27c','x27v','x28o','x28h','x28l','x28c','x28v','x29o','x29h','x29l','x29c','x29v','x30o','x30h','x30l','x30c','x30v','x31o','x31h','x31l','x31c','x31v','x32o','x32h','x32l','x32c','x32v','x33o','x33h','x33l','x33c','x33v','x34o','x34h','x34l','x34c','x34v','x35o','x35h','x35l','x35c','x35v','x36o','x36h','x36l','x36c','x36v','x37o','x37h','x37l','x37c','x37v','x38o','x38h','x38l','x38c','x38v','x39o','x39h','x39l','x39c','x39v','x40o','x40h','x40l','x40c','x40v','x41o','x41h','x41l','x41c','x41v','x42o','x42h','x42l','x42c','x42v','x43o','x43h','x43l','x43c','x43v','x44o','x44h','x44l','x44c','x44v','x45o','x45h','x45l','x45c','x45v','x46o','x46h','x46l','x46c','x46v','x47o','x47h','x47l','x47c','x47v','x48o','x48h','x48l','x48c','x48v','x49o','x49h','x49l','x49c','x49v','x50o','x50h','x50l','x50c','x50v','nextclose']
dataset = pd.read_csv(dataset_path, names=column_names,
na_values = "?", comment='\t',
sep=",", skipinitialspace=True, skiprows = [0])
print('printing dataset tail...')
print(dataset.tail())
train_dataset = dataset.sample(frac=0.8,random_state=0)
test_dataset = dataset.drop(train_dataset.index)
train_labels = train_dataset.pop('nextclose')
test_labels = test_dataset.pop('nextclose')
def norm(x):
return x
# return (x - train_stats['mean']) / train_stats['std']
print('normalizing dataset...')
normed_train_data = norm(train_dataset)
normed_test_data = norm(test_dataset)
def build_model():
print('building the model')
model = keras.Sequential([
layers.Dense(512, activation=tf.nn.relu, input_shape=[len(train_dataset.keys())]),
layers.Dense(512, activation=tf.nn.relu), layers.Dense(256, activation=tf.nn.relu),
layers.Dense(1)
])
return model
def load_model_():
print('loading the model')
# load json and create model
json_file = open(checkpoint_model_json_path, 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights(checkpoint_weights_h5_path)
print("Loaded model from disk")
return loaded_model
if resume_from_checkpoint:
model = load_model_()
else:
model = build_model()
model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mean_absolute_error', 'mean_squared_error', 'accuracy'])
model.summary()
print('testing 10 widthed batch...')
example_batch = normed_train_data[:10]
example_result = model.predict(example_batch)
print(example_result)
def plot_history(history):
hist = pd.DataFrame(history.history)
hist['epoch'] = history.epoch
plt.figure()
plt.xlabel('Epoch')
plt.ylabel('Mean Abs Error [nextclose]')
plt.plot(hist['epoch'], hist['mean_absolute_error'],
label='Train Error')
plt.plot(hist['epoch'], hist['val_mean_absolute_error'],
label = 'Val Error')
plt.ylim([0,5])
plt.legend()
plt.figure()
plt.xlabel('Epoch')
plt.ylabel('Mean Square Error [$nextclose^2$]')
plt.plot(hist['epoch'], hist['mean_squared_error'],
label='Train Error')
plt.plot(hist['epoch'], hist['val_mean_squared_error'],
label = 'Val Error')
plt.ylim([0,20])
plt.legend()
plt.show()
print('fitting the model...')
mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', save_weights_only=True, period=500)
print('saving the model...')
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
history = model.fit(
normed_train_data, train_labels,
epochs=1, validation_split = 0.2, verbose=2,
batch_size=100000, callbacks=[mc])
print('evaluating the model...')
loss, mae, mse, accuracy = model.evaluate(normed_test_data, test_labels, verbose=0)
print("Testing set Mean Abs Error: {:5.2f} nextclose".format(mae))
print("Testing set Accuracy: {:5.2f} nextclose".format(accuracy))
输出：
1.13.1
reading dataset...
printing dataset tail...
paircode x1o x1h x1l x1c x1v x2o x2h x2l x2c x2v x3o x3h ... x48c x48v x49o x49h x49l x49c x49v x50o x50h x50l x50c x50v nextclose
381045 50 112.606 112.622 112.606 112.619 0.0 112.580 112.581 112.561 112.575 0.0 112.601 112.612 ... 112.118 0.0 112.083 112.090 112.079 112.087 0.0 112.025 112.033 112.023 112.032 0.0 112.033
381046 50 112.580 112.581 112.561 112.575 0.0 112.601 112.612 112.598 112.599 0.0 112.581 112.599 ... 112.087 0.0 112.025 112.033 112.023 112.032 0.0 112.031 112.034 112.031 112.033 0.0 112.141
381047 50 112.601 112.612 112.598 112.599 0.0 112.581 112.599 112.580 112.593 0.0 112.548 112.548 ... 112.032 0.0 112.031 112.034 112.031 112.033 0.0 112.142 112.149 112.140 112.141 0.0 112.157
381048 50 112.581 112.599 112.580 112.593 0.0 112.548 112.548 112.540 112.542 0.0 112.551 112.565 ... 112.033 0.0 112.142 112.149 112.140 112.141 0.0 112.161 112.161 112.157 112.157 0.0 112.121
381049 50 112.548 112.548 112.540 112.542 0.0 112.551 112.565 112.551 112.565 0.0 112.564 112.577 ... 112.141 0.0 112.161 112.161 112.157 112.157 0.0 112.121 112.129 112.121 112.121 0.0 112.140
[5 rows x 252 columns]
normalizing dataset...
loading the model
WARNING:tensorflow:From C:\Program Files\Python36\lib\site-packages\tensorflow\python\ops\resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
2019-04-05 12:10:15.520118: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
Loaded model from disk
WARNING:tensorflow:From C:\Program Files\Python36\lib\site-packages\tensorflow\python\keras\utils\losses_utils.py:170: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 512) 129024
_________________________________________________________________
dense_1 (Dense) (None, 512) 262656
_________________________________________________________________
dense_2 (Dense) (None, 256) 131328
_________________________________________________________________
dense_3 (Dense) (None, 1) 257
=================================================================
Total params: 523,265
Trainable params: 523,265
Non-trainable params: 0
_________________________________________________________________
testing 10 widthed batch...
[[106.244064]
[ 76.667534]
[ 82.01627 ]
[ 79.776405]
[116.600204]
[ 95.28444 ]
[ 76.96633 ]
[118.25993 ]
[120.39911 ]
[108.5381 ]]
fitting the model...
saving the model...
Train on 243872 samples, validate on 60968 samples
WARNING:tensorflow:From C:\Program Files\Python36\lib\site-packages\tensorflow\python\ops\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
- 6s - loss: 56.9330 - mean_absolute_error: 5.3921 - mean_squared_error: 56.9330 - acc: 0.0000e+00 - val_loss: 38.9868 - val_mean_absolute_error: 6.1875 - val_mean_squared_error: 38.9868 - val_acc: 0.0000e+00
evaluating the model...
Testing set Mean Abs Error: 6.19 nextclose
Testing set Accuracy: 0.00 nextclose
提前致谢。
您可以简单地构建模型、编译它并保留随机初始化的权重以开始训练。接下来，恢复训练：构建模型，编译它，然后重新加载保存的权重。
对于恢复培训，您不应该 运行 完整代码，
只是 运行 这个 :
model= load_model('model.h5')
history = model.fit(normed_train_data, train_labels, epochs=1, v
validation_split = 0.2, verbose=2,
batch_size=128, callbacks=[mc])`
但是你应该编辑这个：
mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', save_weights_only=True, period=100)
进入这个：
mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', save_weights_only=False, period=100)
如前所述，这是 tensorflow-gpu-nighlybuild 2.0 中已修复的错误 here.
我正在使用 tensorflow.keras 来构建一个具有 3 个密集层的简单神经网络。我能够成功训练模型 9000 个时期，达到 0.0496 的均方误差 (MSE)。不管怎样恢复模型，它在大约 57 MSE 时开始训练。
这可能表明模型权重未成功加载，但是当从头开始重新启动训练过程时（不加载以前保存的权重），MSE 从大约 +9000 开始。
编辑：
- 这是正常问题，还是我做错了什么？
- 为什么即使经过 9000 个迭代后准确率始终为 0.0？
我的代码如下：
from __future__ import absolute_import, division, print_function
import pathlib
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import model_from_json
from tensorflow.keras.models import load_model
print(tf.__version__)
dataset_path = 'D:\data.csv'
checkpoint_model_json_path = 'modelBackup/model.json'
checkpoint_weights_h5_path = 'modelBackup/weights00009000.h5'
resume_from_checkpoint = True
print('reading dataset...')
column_names = ['paircode','x1o','x1h','x1l','x1c','x1v','x2o','x2h','x2l','x2c','x2v','x3o','x3h','x3l','x3c','x3v','x4o','x4h','x4l','x4c','x4v','x5o','x5h','x5l','x5c','x5v','x6o','x6h','x6l','x6c','x6v','x7o','x7h','x7l','x7c','x7v','x8o','x8h','x8l','x8c','x8v','x9o','x9h','x9l','x9c','x9v','x10o','x10h','x10l','x10c','x10v','x11o','x11h','x11l','x11c','x11v','x12o','x12h','x12l','x12c','x12v','x13o','x13h','x13l','x13c','x13v','x14o','x14h','x14l','x14c','x14v','x15o','x15h','x15l','x15c','x15v','x16o','x16h','x16l','x16c','x16v','x17o','x17h','x17l','x17c','x17v','x18o','x18h','x18l','x18c','x18v','x19o','x19h','x19l','x19c','x19v','x20o','x20h','x20l','x20c','x20v','x21o','x21h','x21l','x21c','x21v','x22o','x22h','x22l','x22c','x22v','x23o','x23h','x23l','x23c','x23v','x24o','x24h','x24l','x24c','x24v','x25o','x25h','x25l','x25c','x25v','x26o','x26h','x26l','x26c','x26v','x27o','x27h','x27l','x27c','x27v','x28o','x28h','x28l','x28c','x28v','x29o','x29h','x29l','x29c','x29v','x30o','x30h','x30l','x30c','x30v','x31o','x31h','x31l','x31c','x31v','x32o','x32h','x32l','x32c','x32v','x33o','x33h','x33l','x33c','x33v','x34o','x34h','x34l','x34c','x34v','x35o','x35h','x35l','x35c','x35v','x36o','x36h','x36l','x36c','x36v','x37o','x37h','x37l','x37c','x37v','x38o','x38h','x38l','x38c','x38v','x39o','x39h','x39l','x39c','x39v','x40o','x40h','x40l','x40c','x40v','x41o','x41h','x41l','x41c','x41v','x42o','x42h','x42l','x42c','x42v','x43o','x43h','x43l','x43c','x43v','x44o','x44h','x44l','x44c','x44v','x45o','x45h','x45l','x45c','x45v','x46o','x46h','x46l','x46c','x46v','x47o','x47h','x47l','x47c','x47v','x48o','x48h','x48l','x48c','x48v','x49o','x49h','x49l','x49c','x49v','x50o','x50h','x50l','x50c','x50v','nextclose']
dataset = pd.read_csv(dataset_path, names=column_names,
na_values = "?", comment='\t',
sep=",", skipinitialspace=True, skiprows = [0])
print('printing dataset tail...')
print(dataset.tail())
train_dataset = dataset.sample(frac=0.8,random_state=0)
test_dataset = dataset.drop(train_dataset.index)
train_labels = train_dataset.pop('nextclose')
test_labels = test_dataset.pop('nextclose')
def norm(x):
return x
# return (x - train_stats['mean']) / train_stats['std']
print('normalizing dataset...')
normed_train_data = norm(train_dataset)
normed_test_data = norm(test_dataset)
def build_model():
print('building the model')
model = keras.Sequential([
layers.Dense(512, activation=tf.nn.relu, input_shape=[len(train_dataset.keys())]),
layers.Dense(512, activation=tf.nn.relu), layers.Dense(256, activation=tf.nn.relu),
layers.Dense(1)
])
return model
def load_model_():
print('loading the model')
# load json and create model
json_file = open(checkpoint_model_json_path, 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights(checkpoint_weights_h5_path)
print("Loaded model from disk")
return loaded_model
if resume_from_checkpoint:
model = load_model_()
else:
model = build_model()
model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mean_absolute_error', 'mean_squared_error', 'accuracy'])
model.summary()
print('testing 10 widthed batch...')
example_batch = normed_train_data[:10]
example_result = model.predict(example_batch)
print(example_result)
def plot_history(history):
hist = pd.DataFrame(history.history)
hist['epoch'] = history.epoch
plt.figure()
plt.xlabel('Epoch')
plt.ylabel('Mean Abs Error [nextclose]')
plt.plot(hist['epoch'], hist['mean_absolute_error'],
label='Train Error')
plt.plot(hist['epoch'], hist['val_mean_absolute_error'],
label = 'Val Error')
plt.ylim([0,5])
plt.legend()
plt.figure()
plt.xlabel('Epoch')
plt.ylabel('Mean Square Error [$nextclose^2$]')
plt.plot(hist['epoch'], hist['mean_squared_error'],
label='Train Error')
plt.plot(hist['epoch'], hist['val_mean_squared_error'],
label = 'Val Error')
plt.ylim([0,20])
plt.legend()
plt.show()
print('fitting the model...')
mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', save_weights_only=True, period=500)
print('saving the model...')
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
history = model.fit(
normed_train_data, train_labels,
epochs=1, validation_split = 0.2, verbose=2,
batch_size=100000, callbacks=[mc])
print('evaluating the model...')
loss, mae, mse, accuracy = model.evaluate(normed_test_data, test_labels, verbose=0)
print("Testing set Mean Abs Error: {:5.2f} nextclose".format(mae))
print("Testing set Accuracy: {:5.2f} nextclose".format(accuracy))
输出：
1.13.1
reading dataset...
printing dataset tail...
paircode x1o x1h x1l x1c x1v x2o x2h x2l x2c x2v x3o x3h ... x48c x48v x49o x49h x49l x49c x49v x50o x50h x50l x50c x50v nextclose
381045 50 112.606 112.622 112.606 112.619 0.0 112.580 112.581 112.561 112.575 0.0 112.601 112.612 ... 112.118 0.0 112.083 112.090 112.079 112.087 0.0 112.025 112.033 112.023 112.032 0.0 112.033
381046 50 112.580 112.581 112.561 112.575 0.0 112.601 112.612 112.598 112.599 0.0 112.581 112.599 ... 112.087 0.0 112.025 112.033 112.023 112.032 0.0 112.031 112.034 112.031 112.033 0.0 112.141
381047 50 112.601 112.612 112.598 112.599 0.0 112.581 112.599 112.580 112.593 0.0 112.548 112.548 ... 112.032 0.0 112.031 112.034 112.031 112.033 0.0 112.142 112.149 112.140 112.141 0.0 112.157
381048 50 112.581 112.599 112.580 112.593 0.0 112.548 112.548 112.540 112.542 0.0 112.551 112.565 ... 112.033 0.0 112.142 112.149 112.140 112.141 0.0 112.161 112.161 112.157 112.157 0.0 112.121
381049 50 112.548 112.548 112.540 112.542 0.0 112.551 112.565 112.551 112.565 0.0 112.564 112.577 ... 112.141 0.0 112.161 112.161 112.157 112.157 0.0 112.121 112.129 112.121 112.121 0.0 112.140
[5 rows x 252 columns]
normalizing dataset...
loading the model
WARNING:tensorflow:From C:\Program Files\Python36\lib\site-packages\tensorflow\python\ops\resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
2019-04-05 12:10:15.520118: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
Loaded model from disk
WARNING:tensorflow:From C:\Program Files\Python36\lib\site-packages\tensorflow\python\keras\utils\losses_utils.py:170: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 512) 129024
_________________________________________________________________
dense_1 (Dense) (None, 512) 262656
_________________________________________________________________
dense_2 (Dense) (None, 256) 131328
_________________________________________________________________
dense_3 (Dense) (None, 1) 257
=================================================================
Total params: 523,265
Trainable params: 523,265
Non-trainable params: 0
_________________________________________________________________
testing 10 widthed batch...
[[106.244064]
[ 76.667534]
[ 82.01627 ]
[ 79.776405]
[116.600204]
[ 95.28444 ]
[ 76.96633 ]
[118.25993 ]
[120.39911 ]
[108.5381 ]]
fitting the model...
saving the model...
Train on 243872 samples, validate on 60968 samples
WARNING:tensorflow:From C:\Program Files\Python36\lib\site-packages\tensorflow\python\ops\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
- 6s - loss: 56.9330 - mean_absolute_error: 5.3921 - mean_squared_error: 56.9330 - acc: 0.0000e+00 - val_loss: 38.9868 - val_mean_absolute_error: 6.1875 - val_mean_squared_error: 38.9868 - val_acc: 0.0000e+00
evaluating the model...
Testing set Mean Abs Error: 6.19 nextclose
Testing set Accuracy: 0.00 nextclose
提前致谢。
您可以简单地构建模型、编译它并保留随机初始化的权重以开始训练。接下来，恢复训练：构建模型，编译它，然后重新加载保存的权重。
对于恢复培训，您不应该 运行 完整代码， 只是 运行 这个 :
model= load_model('model.h5')
history = model.fit(normed_train_data, train_labels, epochs=1, v
validation_split = 0.2, verbose=2,
batch_size=128, callbacks=[mc])`
但是你应该编辑这个：
mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', save_weights_only=True, period=100)
进入这个：
mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', save_weights_only=False, period=100)
如前所述，这是 tensorflow-gpu-nighlybuild 2.0 中已修复的错误 here.