Tensorflow 权重初始化
Tensorflow weight initialization
关于MNIST tutorial on the TensorFlow website, I ran an experiment (gist) to see what the effect of different weight initializations would be on learning. I noticed that, against what I read in the popular [Xavier, Glorot 2010] paper,无论权重初始化如何,学习都很好。
不同的曲线代表 w
的不同值,用于初始化卷积层和全连接层的权重。请注意,w
的所有值都可以正常工作,即使 0.3
和 1.0
最终性能较低并且某些值训练得更快 - 特别是 0.03
和 0.1
最快。尽管如此,该图显示了相当大的 w
范围,这表明 'robustness' w.r.t。权重初始化。
def weight_variable(shape, w=0.1):
initial = tf.truncated_normal(shape, stddev=w)
return tf.Variable(initial)
def bias_variable(shape, w=0.1):
initial = tf.constant(w, shape=shape)
return tf.Variable(initial)
问题:为什么这个网络没有梯度消失或爆炸问题?
我建议您阅读要点以了解实现细节,但这里是供参考的代码。在我的 Nvidia 960m 上花了大约一个小时,尽管我想它也可以在 合理的 时间内 运行 在 CPU 上。
import time
from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf
from tensorflow.python.client import device_lib
import numpy
import matplotlib.pyplot as pyplot
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
# Weight initialization
def weight_variable(shape, w=0.1):
initial = tf.truncated_normal(shape, stddev=w)
return tf.Variable(initial)
def bias_variable(shape, w=0.1):
initial = tf.constant(w, shape=shape)
return tf.Variable(initial)
# Network architecture
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
def build_network_for_weight_initialization(w):
""" Builds a CNN for the MNIST-problem:
- 32 5x5 kernels convolutional layer with bias and ReLU activations
- 2x2 maxpooling
- 64 5x5 kernels convolutional layer with bias and ReLU activations
- 2x2 maxpooling
- Fully connected layer with 1024 nodes + bias and ReLU activations
- dropout
- Fully connected softmax layer for classification (of 10 classes)
Returns the x, and y placeholders for the train data, the output
of the network and the dropbout placeholder as a tuple of 4 elements.
"""
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
x_image = tf.reshape(x, [-1,28,28,1])
W_conv1 = weight_variable([5, 5, 1, 32], w)
b_conv1 = bias_variable([32], w)
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
W_conv2 = weight_variable([5, 5, 32, 64], w)
b_conv2 = bias_variable([64], w)
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
W_fc1 = weight_variable([7 * 7 * 64, 1024], w)
b_fc1 = bias_variable([1024], w)
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
W_fc2 = weight_variable([1024, 10], w)
b_fc2 = bias_variable([10], w)
y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
return (x, y_, y_conv, keep_prob)
# Experiment
def evaluate_for_weight_init(w):
""" Returns an accuracy learning curve for a network trained on
10000 batches of 50 samples. The learning curve has one item
every 100 batches."""
with tf.Session() as sess:
x, y_, y_conv, keep_prob = build_network_for_weight_initialization(w)
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess.run(tf.global_variables_initializer())
lr = []
for _ in range(100):
for i in range(100):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
assert mnist.test.images.shape[0] == 10000
# This way the accuracy-evaluation fits in my 2GB laptop GPU.
a = sum(
accuracy.eval(feed_dict={
x: mnist.test.images[2000*i:2000*(i+1)],
y_: mnist.test.labels[2000*i:2000*(i+1)],
keep_prob: 1.0})
for i in range(5)) / 5
lr.append(a)
return lr
ws = [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0]
accuracies = [
[evaluate_for_weight_init(w) for w in ws]
for _ in range(3)
]
# Plotting results
pyplot.plot(numpy.array(accuracies).mean(0).T)
pyplot.ylim(0.9, 1)
pyplot.xlim(0,140)
pyplot.xlabel('batch (x 100)')
pyplot.ylabel('test accuracy')
pyplot.legend(ws)
Logistic 函数更容易出现梯度消失,因为它们的梯度都小于 1,所以在反向传播过程中你乘的越多,你的梯度就会变得越小(而且很快),而 RelU 有一个梯度1 的积极部分,所以它没有这个问题。
另外,你的人脉还不够深,不会受到影响。
权重初始化策略可能是改进模型的一个重要且经常被忽视的步骤,因为这是现在 Google 上的最高结果,我认为它可以保证更详细的答案。
一般来说,每一层的激活函数梯度、incoming/outgoing个连接数(fan_in/fan_out)和权重方差的总和应该等于1。这样,当您通过网络反向传播时,输入和输出梯度之间的方差将保持一致,并且您不会遭受梯度爆炸或消失的影响。即使 ReLU 对 exploding/vanishing 梯度的抵抗力更强,您可能仍然会遇到问题。
OP 使用的 tf.truncated_normal 进行随机初始化,鼓励更新权重 "differently",但 未 考虑上述优化策略。在较小的网络上,这可能不是问题,但如果您想要更深的网络或更快的训练时间,那么您最好尝试基于最近研究的权重初始化策略。
对于 ReLU 函数之前的权重,您可以使用默认设置:
tf.contrib.layers.variance_scaling_initializer
for tanh/sigmoid 激活层 "xavier" 可能更合适:
tf.contrib.layers.xavier_initializer
有关这些功能和相关论文的更多详细信息,请访问:
https://www.tensorflow.org/versions/r0.12/api_docs/python/contrib.layers/initializers
除了权重初始化策略,进一步的优化可以探索批量归一化:https://www.tensorflow.org/api_docs/python/tf/nn/batch_normalization
关于MNIST tutorial on the TensorFlow website, I ran an experiment (gist) to see what the effect of different weight initializations would be on learning. I noticed that, against what I read in the popular [Xavier, Glorot 2010] paper,无论权重初始化如何,学习都很好。
不同的曲线代表 w
的不同值,用于初始化卷积层和全连接层的权重。请注意,w
的所有值都可以正常工作,即使 0.3
和 1.0
最终性能较低并且某些值训练得更快 - 特别是 0.03
和 0.1
最快。尽管如此,该图显示了相当大的 w
范围,这表明 'robustness' w.r.t。权重初始化。
def weight_variable(shape, w=0.1):
initial = tf.truncated_normal(shape, stddev=w)
return tf.Variable(initial)
def bias_variable(shape, w=0.1):
initial = tf.constant(w, shape=shape)
return tf.Variable(initial)
问题:为什么这个网络没有梯度消失或爆炸问题?
我建议您阅读要点以了解实现细节,但这里是供参考的代码。在我的 Nvidia 960m 上花了大约一个小时,尽管我想它也可以在 合理的 时间内 运行 在 CPU 上。
import time
from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf
from tensorflow.python.client import device_lib
import numpy
import matplotlib.pyplot as pyplot
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
# Weight initialization
def weight_variable(shape, w=0.1):
initial = tf.truncated_normal(shape, stddev=w)
return tf.Variable(initial)
def bias_variable(shape, w=0.1):
initial = tf.constant(w, shape=shape)
return tf.Variable(initial)
# Network architecture
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
def build_network_for_weight_initialization(w):
""" Builds a CNN for the MNIST-problem:
- 32 5x5 kernels convolutional layer with bias and ReLU activations
- 2x2 maxpooling
- 64 5x5 kernels convolutional layer with bias and ReLU activations
- 2x2 maxpooling
- Fully connected layer with 1024 nodes + bias and ReLU activations
- dropout
- Fully connected softmax layer for classification (of 10 classes)
Returns the x, and y placeholders for the train data, the output
of the network and the dropbout placeholder as a tuple of 4 elements.
"""
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
x_image = tf.reshape(x, [-1,28,28,1])
W_conv1 = weight_variable([5, 5, 1, 32], w)
b_conv1 = bias_variable([32], w)
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
W_conv2 = weight_variable([5, 5, 32, 64], w)
b_conv2 = bias_variable([64], w)
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
W_fc1 = weight_variable([7 * 7 * 64, 1024], w)
b_fc1 = bias_variable([1024], w)
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
W_fc2 = weight_variable([1024, 10], w)
b_fc2 = bias_variable([10], w)
y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
return (x, y_, y_conv, keep_prob)
# Experiment
def evaluate_for_weight_init(w):
""" Returns an accuracy learning curve for a network trained on
10000 batches of 50 samples. The learning curve has one item
every 100 batches."""
with tf.Session() as sess:
x, y_, y_conv, keep_prob = build_network_for_weight_initialization(w)
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess.run(tf.global_variables_initializer())
lr = []
for _ in range(100):
for i in range(100):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
assert mnist.test.images.shape[0] == 10000
# This way the accuracy-evaluation fits in my 2GB laptop GPU.
a = sum(
accuracy.eval(feed_dict={
x: mnist.test.images[2000*i:2000*(i+1)],
y_: mnist.test.labels[2000*i:2000*(i+1)],
keep_prob: 1.0})
for i in range(5)) / 5
lr.append(a)
return lr
ws = [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0]
accuracies = [
[evaluate_for_weight_init(w) for w in ws]
for _ in range(3)
]
# Plotting results
pyplot.plot(numpy.array(accuracies).mean(0).T)
pyplot.ylim(0.9, 1)
pyplot.xlim(0,140)
pyplot.xlabel('batch (x 100)')
pyplot.ylabel('test accuracy')
pyplot.legend(ws)
Logistic 函数更容易出现梯度消失,因为它们的梯度都小于 1,所以在反向传播过程中你乘的越多,你的梯度就会变得越小(而且很快),而 RelU 有一个梯度1 的积极部分,所以它没有这个问题。
另外,你的人脉还不够深,不会受到影响。
权重初始化策略可能是改进模型的一个重要且经常被忽视的步骤,因为这是现在 Google 上的最高结果,我认为它可以保证更详细的答案。
一般来说,每一层的激活函数梯度、incoming/outgoing个连接数(fan_in/fan_out)和权重方差的总和应该等于1。这样,当您通过网络反向传播时,输入和输出梯度之间的方差将保持一致,并且您不会遭受梯度爆炸或消失的影响。即使 ReLU 对 exploding/vanishing 梯度的抵抗力更强,您可能仍然会遇到问题。
OP 使用的tf.truncated_normal 进行随机初始化,鼓励更新权重 "differently",但 未 考虑上述优化策略。在较小的网络上,这可能不是问题,但如果您想要更深的网络或更快的训练时间,那么您最好尝试基于最近研究的权重初始化策略。
对于 ReLU 函数之前的权重,您可以使用默认设置:
tf.contrib.layers.variance_scaling_initializer
for tanh/sigmoid 激活层 "xavier" 可能更合适:
tf.contrib.layers.xavier_initializer
有关这些功能和相关论文的更多详细信息,请访问: https://www.tensorflow.org/versions/r0.12/api_docs/python/contrib.layers/initializers
除了权重初始化策略,进一步的优化可以探索批量归一化:https://www.tensorflow.org/api_docs/python/tf/nn/batch_normalization