keras密集模型的矩阵逆逼近

Question

我正在训练神经网络来计算 3x3 矩阵的逆矩阵。我正在使用具有 1 层和 9 个神经元的 Keras 密集模型。第一层的激活函数是 'relu' 并且在输出层是线性的。我正在使用 10000 个行列式 1 的矩阵。我得到的结果不是很好（RMSE 为数百）。我一直在尝试更多的层数、更多的神经元和其他激活函数，但收获很小。这是代码：

import numpy as np
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense

def generator(nb_samples, matrix_size = 2, entries_range = (0,1), determinant = None):
    ''' 
    Generate nb_samples random matrices of size matrix_size with float 
    entries in interval entries_range and of determinant determinant 
    ''' 
    matrices = []
    if determinant:
        inverses = []
        for i in range(nb_samples):
            matrix = np.random.uniform(entries_range[0], entries_range[1], (matrix_size,matrix_size))
            matrix[0] *= determinant/np.linalg.det(matrix)
            matrices.append(matrix.reshape(matrix_size**2,))
            inverses.append(np.array(np.linalg.inv(matrix)).reshape(matrix_size**2,))
        return np.array(matrices), np.array(inverses)
    else:
        determinants = []
        for i in range(nb_samples):
            matrix = np.random.uniform(entries_range[0], entries_range[1], (matrix_size,matrix_size))
            determinants.append(np.array(np.linalg.det(matrix)).reshape(1,))
            matrices.append(matrix.reshape(matrix_size**2,))    
        return np.array(matrices), np.array(determinants)

### Select number of samples, matrix size and range of entries in matrices
nb_samples = 10000
matrix_size = 3
entries_range = (0, 100)
determinant = 1

### Generate random matrices and determinants
matrices, inverses = generator(nb_samples, matrix_size = matrix_size, entries_range = entries_range, determinant = determinant)

### Select number of layers and neurons
nb_hidden_layers = 1
nb_neurons = matrix_size**2
activation = 'relu'

### Create dense neural network with nb_hidden_layers hidden layers having nb_neurons neurons each
model = Sequential()
model.add(Dense(nb_neurons, input_dim = matrix_size**2, activation = activation))
for i in range(nb_hidden_layers):
    model.add(Dense(nb_neurons, activation = activation))
model.add(Dense(matrix_size**2))
model.compile(loss='mse', optimizer='adam')

### Train and save model using train size of 0.66
history = model.fit(matrices, inverses, epochs = 400, batch_size = 100, verbose = 0, validation_split = 0.33)

### Get validation loss from object 'history'
rmse = np.sqrt(history.history['val_loss'][-1])

### Print RMSE and parameter values
print('''
Validation RMSE: {}
Number of hidden layers: {}
Number of neurons: {}
Number of samples: {}
Matrices size: {}
Range of entries: {}
Determinant: {}
'''.format(rmse,nb_hidden_layers,nb_neurons,nb_samples,matrix_size,entries_range,determinant))

网上查了一下，好像有论文是讲逆矩阵逼近问题的。但是，在更改模型之前，我想知道是否还有其他我可以更改的参数可能会对错误产生更大的影响。我希望有人可以提供一些见解。谢谢。

Answer 1

反转 3x3 矩阵对于神经网络来说非常困难，因为它们往往不擅长乘法或除法激活。我无法让它与简单的密集网络一起工作，但 7 层 resnet 可以解决问题。它有数百万个权重，所以它需要超过 10000 个示例：我发现它完全记住了多达 100,000 个样本，即使有 10,000,000 个样本也严重过拟合，所以我只是连续生成样本并在生成每个样本时将其馈送到网络一次.

import tensorflow as tf
import numpy as np

import matplotlib.pyplot as plt

#too_small_model = tf.keras.Sequential([
#    tf.keras.layers.Flatten(),
#    tf.keras.layers.Dense(1500, activation="relu"),
#    tf.keras.layers.Dense(1500, activation="relu"), 
#    tf.keras.layers.Dense(N * N),
#    tf.keras.layers.Reshape([ N, N])
#])

N = 3

inp = tf.keras.layers.Input(shape=[N, N])
x = tf.keras.layers.Flatten()(inp)
x = tf.keras.layers.Dense(128, activation="relu")(x)
for _ in range(7):
    skip = x
    for _ in range(4):
        y = tf.keras.layers.Dense(256, activation="relu")(x)
        x = tf.keras.layers.concatenate([x, y])
    #x = tf.keras.layers.BatchNormalization()(x)
    x = tf.keras.layers.Dense(128,
        kernel_initializer=tf.keras.initializers.Zeros(),
        bias_initializer=tf.keras.initializers.Zeros()
    )(x)
    x = skip + x
    #x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Dense(N * N)(x)
x = tf.keras.layers.Reshape([N, N])(x)
model2 = tf.keras.models.Model(inp, x)

model2.compile(loss="mean_squared_error", optimizer=tf.keras.optimizers.Adam(learning_rate=.00001))

for _ in range(5000):
    random_matrices = np.random.random((1000000, N, N)) * 4 - 2
    random_matrices = random_matrices[np.abs(np.linalg.det(random_matrices)) > .1]
    inverses = np.linalg.inv(random_matrices)
    inverses = inverses / 5. # normalize target values, large target values hamper training
    model2.fit(random_matrices, inverses, epochs=1, batch_size=1024)

zz = model2.predict(random_matrices[:10000])

plt.scatter(inverses[:10000], zz, s=.0001)

print(random_matrices[76] @ zz[76] * 5)

keras密集模型的矩阵逆逼近

Matrix inverse approximation with keras dense model

python

matrix

neural-network

multi-layer

keras