在 tensorflow 教程中训练深度神经网络时的 nan loss

Question

我正在尝试在 notMNIST 上训练一个具有 1 个以上隐藏层的神经网络。当我有一个隐藏层时它工作正常，但是当我添加多个隐藏层时我开始得到 nan 作为损失。这是我使用的代码

from __future__ import print_function
import numpy as np
import tensorflow as tf
from six.moves import cPickle as pickle
from six.moves import range

batch_size = 128
num_hidden = 1024
num_hidden2 = 300
num_hidden3 = 50
SEED = 1234567
keep_prob = 0.5

graph1 = tf.Graph()
with graph1.as_default():

  # Input data. For the training data, we use a placeholder that will be fed
  # at run time with a training minibatch.
  tf_train_dataset = tf.placeholder(tf.float32,
                                    shape=(batch_size, image_size * image_size))
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
  tf_valid_dataset = tf.constant(valid_dataset)
  tf_test_dataset = tf.constant(test_dataset)

  # Variables.
  weights1 = tf.Variable(tf.truncated_normal([image_size * image_size, num_hidden]))
  biases1 = tf.Variable(tf.zeros([num_hidden]))

  weights2 = tf.Variable(tf.truncated_normal([num_hidden, num_hidden2]))
  biases2 = tf.Variable(tf.zeros([num_hidden2]))

  weights3 = tf.Variable(tf.truncated_normal([num_hidden2, num_hidden3]))
  biases3 = tf.Variable(tf.zeros([num_hidden3]))

  weights4 = tf.Variable(tf.truncated_normal([num_hidden3, num_labels]))
  biases4 = tf.Variable(tf.zeros([num_labels]))

  # Training computation.
  l1 = tf.matmul(tf_train_dataset, weights1) + biases1
  h1 = tf.nn.relu(l1)
  h1 = tf.nn.dropout(h1, 0.5, seed=SEED)

  l2 = tf.matmul(h1, weights2) + biases2
  h2 = tf.nn.relu(l2) 
  h2 = tf.nn.dropout(h2, 0.5, seed=SEED)

  l3 = tf.matmul(h2, weights3) + biases3
  h3 = tf.nn.relu(l3) 
  h3 = tf.nn.dropout(h3, 0.5, seed=SEED)

  logits = tf.matmul(h3, weights4) + biases4


  loss = tf.reduce_mean(
    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))

  # L2 regularization for the fully connected parameters.
  regularizers = (tf.nn.l2_loss(weights1) + tf.nn.l2_loss(biases1) +
                  tf.nn.l2_loss(weights2) + tf.nn.l2_loss(biases2) +
                  tf.nn.l2_loss(weights3) + tf.nn.l2_loss(biases3) +
                  tf.nn.l2_loss(weights4) + tf.nn.l2_loss(biases4))
  # Add the regularization term to the loss.
  loss += 5e-4 * regularizers

  # Optimizer.
  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)

  # Predictions for the training, validation, and test data.
  train_prediction = tf.nn.softmax(logits)

  v_l1 = tf.matmul(tf_valid_dataset, weights1) + biases1
  v_h1 = tf.nn.relu(v_l1)

  v_l2 = tf.matmul(v_h1, weights2) + biases2
  v_h2 = tf.nn.relu(v_l2) 

  v_l3 = tf.matmul(v_h2, weights3) + biases3
  v_h3 = tf.nn.relu(v_l3) 

  v_logits = tf.matmul(v_h3, weights4) + biases4
  valid_prediction = tf.nn.softmax(v_logits)


  t_l1 = tf.matmul(tf_test_dataset, weights1) + biases1
  t_h1 = tf.nn.relu(t_l1)

  t_l2 = tf.matmul(t_h1, weights2) + biases2
  t_h2 = tf.nn.relu(t_l2) 

  t_l3 = tf.matmul(t_h2, weights3) + biases3
  t_h3 = tf.nn.relu(t_l3) 

  t_logits = tf.matmul(t_h3, weights4) + biases4
  test_prediction = tf.nn.softmax(t_logits)


num_steps = 3001

with tf.Session(graph=graph1) as session:
  tf.initialize_all_variables().run()
  print("Initialized")
  for step in range(num_steps):
    # Pick an offset within the training data, which has been randomized.
    # Note: we could use better randomization across epochs.
    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
    # Generate a minibatch.
    batch_data = train_dataset[offset:(offset + batch_size), :]
    batch_labels = train_labels[offset:(offset + batch_size), :]
    # Prepare a dictionary telling the session where to feed the minibatch.
    # The key of the dictionary is the placeholder node of the graph to be fed,
    # and the value is the numpy array to feed to it.
    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
    _, l, predictions = session.run(
      [optimizer, loss, train_prediction], feed_dict=feed_dict)
    if (step % 500 == 0):
      print("Minibatch loss at step %d: %f" % (step, l))
      print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
      print("Validation accuracy: %.1f%%" % accuracy(
        valid_prediction.eval(), valid_labels))
  print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))

这是我得到的输出

Initialized
Minibatch loss at step 0: 48759.078125
Minibatch accuracy: 10.2%
Validation accuracy: 10.0%
Minibatch loss at step 500: nan
Minibatch accuracy: 9.4%
Validation accuracy: 10.0%
Minibatch loss at step 1000: nan
Minibatch accuracy: 8.6%
Validation accuracy: 10.0%
Minibatch loss at step 1500: nan
Minibatch accuracy: 11.7%
Validation accuracy: 10.0%
Minibatch loss at step 2000: nan
Minibatch accuracy: 6.2%
Validation accuracy: 10.0%
Minibatch loss at step 2500: nan
Minibatch accuracy: 10.2%
Validation accuracy: 10.0%
Minibatch loss at step 3000: nan
Minibatch accuracy: 7.8%
Validation accuracy: 10.0%
Test accuracy: 10.0%

Answer 1

尝试降低权重的标准差。默认设置为 1。它对我有用。

在 tensorflow 教程中训练深度神经网络时的 nan loss

nan loss when training a deep neural network in tensorflow tutorial

machine-learning

backpropagation

neural-network

deep-learning

tensorflow