DeepLearning4J：形状与前馈自动编码器不匹配
DeepLearning4J: Shapes do not match on FeedForward Auto Encoder
我正在实施一个自动编码器来检测物联网传感器数据的异常情况。我的数据集来自模拟，但基本上它是加速度计数据 - 三个维度，每个轴一个。
我正在从 CSV 文件中读取它，第 2-4 列包含数据 - 抱歉代码质量，它又快又脏：
private static DataSetIterator getTrainingData(int batchSize, Random rand) {
double[] ix = new double[nSamples];
double[] iy = new double[nSamples];
double[] iz = new double[nSamples];
double[] ox = new double[nSamples];
double[] oy = new double[nSamples];
double[] oz = new double[nSamples];
Reader in;
try {
in = new FileReader("/Users/romeokienzler/Downloads/lorenz_healthy.csv");
Iterable<CSVRecord> records;
records = CSVFormat.DEFAULT.parse(in);
int index = 0;
for (CSVRecord record : records) {
String[] recordArray = record.get(0).split(";");
ix[index] = Double.parseDouble(recordArray[1]);
iy[index] = Double.parseDouble(recordArray[2]);
iz[index] = Double.parseDouble(recordArray[3]);
ox[index] = Double.parseDouble(recordArray[1]);
oy[index] = Double.parseDouble(recordArray[2]);
oz[index] = Double.parseDouble(recordArray[3]);
index++;
}
INDArray ixNd = Nd4j.create(ix);
INDArray iyNd = Nd4j.create(iy);
INDArray izNd = Nd4j.create(iz);
INDArray oxNd = Nd4j.create(ox);
INDArray oyNd = Nd4j.create(oy);
INDArray ozNd = Nd4j.create(oz);
INDArray iNd = Nd4j.hstack(ixNd, iyNd, izNd);
INDArray oNd = Nd4j.hstack(oxNd, oyNd, ozNd);
DataSet dataSet = new DataSet(iNd, oNd);
List<DataSet> listDs = dataSet.asList();
Collections.shuffle(listDs, rng);
return new ListDataSetIterator(listDs, batchSize);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(-1);
return null;
}
}
这是网络：
public static void main(String[] args) {
// Generate the training data
DataSetIterator iterator = getTrainingData(batchSize, rng);
// Create the network
int numInput = 3;
int numOutputs = 3;
int nHidden = 1;
int listenerFreq = batchSize / 5;
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed)
.gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue)
.gradientNormalizationThreshold(1.0).iterations(iterations).momentum(0.5)
.momentumAfter(Collections.singletonMap(3, 0.9))
.optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list(2)
.layer(0,
new AutoEncoder.Builder().nIn(numInput).nOut(nHidden).weightInit(WeightInit.XAVIER)
.lossFunction(LossFunction.RMSE_XENT).corruptionLevel(0.3).build())
.layer(1, new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD).activation("softmax").nIn(nHidden)
.nOut(numOutputs).build())
.pretrain(true).backprop(false).build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
model.init();
model.setListeners(Collections.singletonList((IterationListener) new ScoreIterationListener(listenerFreq)));
for (int i = 0; i < nEpochs; i++) {
iterator.reset();
model.fit(iterator);
}
}
我收到以下错误：
形状不匹配：x.shape=[1, 9000], y.shape=[1, 3]
Exception in thread "main" java.lang.IllegalArgumentException: Shapes do not match: x.shape=[1, 9000], y.shape=[1, 3]
at org.nd4j.linalg.api.parallel.tasks.cpu.CPUTaskFactory.getTransformAction(CPUTaskFactory.java:92)
at org.nd4j.linalg.api.ops.executioner.DefaultOpExecutioner.doTransformOp(DefaultOpExecutioner.java:409)
at org.nd4j.linalg.api.ops.executioner.DefaultOpExecutioner.exec(DefaultOpExecutioner.java:62)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2660)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2641)
at org.nd4j.linalg.api.ndarray.BaseNDArray.sub(BaseNDArray.java:2419)
at org.deeplearning4j.nn.layers.feedforward.autoencoder.AutoEncoder.computeGradientAndScore(AutoEncoder.java:123)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:132)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.optimize(BaseOptimizer.java:151)
at org.deeplearning4j.optimize.Solver.optimize(Solver.java:52)
at org.deeplearning4j.nn.layers.BaseLayer.fit(BaseLayer.java:486)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.pretrain(MultiLayerNetwork.java:170)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1134)
at org.deeplearning4j
.examples.feedforward.autoencoder.AnomalyDetector.main(AnomalyDetector.java:136)
但我没有在任何地方定义维度，恕我直言，输入和输出的维度应该是 (3,3000) 和 (3,3000)。我的错误在哪里？
非常感谢...
编辑：更新到最新版本 13.9.16
我遇到了同样的错误（语义上），这是我现在正在做的：
private static DataSetIterator getTrainingData(int batchSize, Random rand) {
double[] ix = new double[nSamples];
double[] iy = new double[nSamples];
double[] iz = new double[nSamples];
double[] ox = new double[nSamples];
double[] oy = new double[nSamples];
double[] oz = new double[nSamples];
try {
RandomAccessFile in = new RandomAccessFile(new File("/Users/romeokienzler/Downloads/lorenz_healthy.csv"),
"r");
int index = 0;
String record;
while ((record = in.readLine()) != null) {
String[] recordArray = record.split(";");
ix[index] = Double.parseDouble(recordArray[1]);
iy[index] = Double.parseDouble(recordArray[2]);
iz[index] = Double.parseDouble(recordArray[3]);
ox[index] = Double.parseDouble(recordArray[1]);
oy[index] = Double.parseDouble(recordArray[2]);
oz[index] = Double.parseDouble(recordArray[3]);
index++;
}
INDArray ixNd = Nd4j.create(ix);
INDArray iyNd = Nd4j.create(iy);
INDArray izNd = Nd4j.create(iz);
INDArray oxNd = Nd4j.create(ox);
INDArray oyNd = Nd4j.create(oy);
INDArray ozNd = Nd4j.create(oz);
INDArray iNd = Nd4j.hstack(ixNd, iyNd, izNd);
INDArray oNd = Nd4j.hstack(oxNd, oyNd, ozNd);
DataSet dataSet = new DataSet(iNd, oNd);
List<DataSet> listDs = dataSet.asList();
Collections.shuffle(listDs, rng);
return new ListDataSetIterator(listDs, batchSize);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(-1);
return null;
}
}
这里是网络：
// Set up network. 784 in/out (as MNIST images are 28x28).
// 784 -> 250 -> 10 -> 250 -> 784
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).iterations(1)
.weightInit(WeightInit.XAVIER).updater(Updater.ADAGRAD).activation("relu")
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(learningRate)
.regularization(true).l2(0.0001).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(1).build())
.layer(1, new OutputLayer.Builder().nIn(1).nOut(3).lossFunction(LossFunctions.LossFunction.MSE).build())
.pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.setListeners(Collections.singletonList((IterationListener) new ScoreIterationListener(1)));
// Load data and split into training and testing sets. 40000 train,
// 10000 test
DataSetIterator iter = getTrainingData(batchSize, rng);
// Train model:
int nEpochs = 30;
while (iter.hasNext()) {
DataSet ds = iter.next();
for (int epoch = 0; epoch < nEpochs; epoch++) {
net.fit(ds.getFeatures(), ds.getLabels());
System.out.println("Epoch " + epoch + " complete");
}
}
我的错误是：
Exception in thread "main" java.lang.IllegalStateException: Mis matched lengths: [9000] != [3]
at org.nd4j.linalg.util.LinAlgExceptions.assertSameLength(LinAlgExceptions.java:39)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2786)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2767)
at org.nd4j.linalg.api.ndarray.BaseNDArray.sub(BaseNDArray.java:2547)
at org.deeplearning4j.nn.layers.BaseOutputLayer.getGradientsAndDelta(BaseOutputLayer.java:182)
at org.deeplearning4j.nn.layers.BaseOutputLayer.backpropGradient(BaseOutputLayer.java:161)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.calcBackpropGradients(MultiLayerNetwork.java:1125)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.backprop(MultiLayerNetwork.java:1077)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(MultiLayerNetwork.java:1817)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:152)
at org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(StochasticGradientDescent.java:54)
at org.deeplearning4j.optimize.Solver.optimize(Solver.java:51)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1445)
at org.deeplearning4j.examples.feedforward.anomalydetection.IoTAnomalyExample.main(IoTAnomalyExample.java:110)
我很确定我弄乱了训练数据 - 训练数据的形状是 3000 行，3 列 - 与目标相同（完全相同的数据，因为我想构建一个自动编码器） - 测试数据可以在这里找到：
https://pmqsimulator-romeokienzler-2310.mybluemix.net/data
有什么想法吗？
感谢Alex Black of Skymind，这就是解决方案（弄错了形状）
INDArray ixNd = Nd4j.create(ix, new int[]{3000,1});
INDArray iyNd = Nd4j.create(iy, new int[]{3000,1});
INDArray izNd = Nd4j.create(iz, new int[]{3000,1});
INDArray oxNd = Nd4j.create(ox, new int[]{3000,1});
INDArray oyNd = Nd4j.create(oy, new int[]{3000,1});
INDArray ozNd = Nd4j.create(oz, new int[]{3000,1});
我正在实施一个自动编码器来检测物联网传感器数据的异常情况。我的数据集来自模拟，但基本上它是加速度计数据 - 三个维度，每个轴一个。
我正在从 CSV 文件中读取它，第 2-4 列包含数据 - 抱歉代码质量，它又快又脏：
private static DataSetIterator getTrainingData(int batchSize, Random rand) {
double[] ix = new double[nSamples];
double[] iy = new double[nSamples];
double[] iz = new double[nSamples];
double[] ox = new double[nSamples];
double[] oy = new double[nSamples];
double[] oz = new double[nSamples];
Reader in;
try {
in = new FileReader("/Users/romeokienzler/Downloads/lorenz_healthy.csv");
Iterable<CSVRecord> records;
records = CSVFormat.DEFAULT.parse(in);
int index = 0;
for (CSVRecord record : records) {
String[] recordArray = record.get(0).split(";");
ix[index] = Double.parseDouble(recordArray[1]);
iy[index] = Double.parseDouble(recordArray[2]);
iz[index] = Double.parseDouble(recordArray[3]);
ox[index] = Double.parseDouble(recordArray[1]);
oy[index] = Double.parseDouble(recordArray[2]);
oz[index] = Double.parseDouble(recordArray[3]);
index++;
}
INDArray ixNd = Nd4j.create(ix);
INDArray iyNd = Nd4j.create(iy);
INDArray izNd = Nd4j.create(iz);
INDArray oxNd = Nd4j.create(ox);
INDArray oyNd = Nd4j.create(oy);
INDArray ozNd = Nd4j.create(oz);
INDArray iNd = Nd4j.hstack(ixNd, iyNd, izNd);
INDArray oNd = Nd4j.hstack(oxNd, oyNd, ozNd);
DataSet dataSet = new DataSet(iNd, oNd);
List<DataSet> listDs = dataSet.asList();
Collections.shuffle(listDs, rng);
return new ListDataSetIterator(listDs, batchSize);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(-1);
return null;
}
}
这是网络：
public static void main(String[] args) {
// Generate the training data
DataSetIterator iterator = getTrainingData(batchSize, rng);
// Create the network
int numInput = 3;
int numOutputs = 3;
int nHidden = 1;
int listenerFreq = batchSize / 5;
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed)
.gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue)
.gradientNormalizationThreshold(1.0).iterations(iterations).momentum(0.5)
.momentumAfter(Collections.singletonMap(3, 0.9))
.optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list(2)
.layer(0,
new AutoEncoder.Builder().nIn(numInput).nOut(nHidden).weightInit(WeightInit.XAVIER)
.lossFunction(LossFunction.RMSE_XENT).corruptionLevel(0.3).build())
.layer(1, new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD).activation("softmax").nIn(nHidden)
.nOut(numOutputs).build())
.pretrain(true).backprop(false).build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
model.init();
model.setListeners(Collections.singletonList((IterationListener) new ScoreIterationListener(listenerFreq)));
for (int i = 0; i < nEpochs; i++) {
iterator.reset();
model.fit(iterator);
}
}
我收到以下错误： 形状不匹配：x.shape=[1, 9000], y.shape=[1, 3]
Exception in thread "main" java.lang.IllegalArgumentException: Shapes do not match: x.shape=[1, 9000], y.shape=[1, 3]
at org.nd4j.linalg.api.parallel.tasks.cpu.CPUTaskFactory.getTransformAction(CPUTaskFactory.java:92)
at org.nd4j.linalg.api.ops.executioner.DefaultOpExecutioner.doTransformOp(DefaultOpExecutioner.java:409)
at org.nd4j.linalg.api.ops.executioner.DefaultOpExecutioner.exec(DefaultOpExecutioner.java:62)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2660)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2641)
at org.nd4j.linalg.api.ndarray.BaseNDArray.sub(BaseNDArray.java:2419)
at org.deeplearning4j.nn.layers.feedforward.autoencoder.AutoEncoder.computeGradientAndScore(AutoEncoder.java:123)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:132)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.optimize(BaseOptimizer.java:151)
at org.deeplearning4j.optimize.Solver.optimize(Solver.java:52)
at org.deeplearning4j.nn.layers.BaseLayer.fit(BaseLayer.java:486)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.pretrain(MultiLayerNetwork.java:170)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1134)
at org.deeplearning4j
.examples.feedforward.autoencoder.AnomalyDetector.main(AnomalyDetector.java:136)
但我没有在任何地方定义维度，恕我直言，输入和输出的维度应该是 (3,3000) 和 (3,3000)。我的错误在哪里？
非常感谢...
编辑：更新到最新版本 13.9.16 我遇到了同样的错误（语义上），这是我现在正在做的：
private static DataSetIterator getTrainingData(int batchSize, Random rand) {
double[] ix = new double[nSamples];
double[] iy = new double[nSamples];
double[] iz = new double[nSamples];
double[] ox = new double[nSamples];
double[] oy = new double[nSamples];
double[] oz = new double[nSamples];
try {
RandomAccessFile in = new RandomAccessFile(new File("/Users/romeokienzler/Downloads/lorenz_healthy.csv"),
"r");
int index = 0;
String record;
while ((record = in.readLine()) != null) {
String[] recordArray = record.split(";");
ix[index] = Double.parseDouble(recordArray[1]);
iy[index] = Double.parseDouble(recordArray[2]);
iz[index] = Double.parseDouble(recordArray[3]);
ox[index] = Double.parseDouble(recordArray[1]);
oy[index] = Double.parseDouble(recordArray[2]);
oz[index] = Double.parseDouble(recordArray[3]);
index++;
}
INDArray ixNd = Nd4j.create(ix);
INDArray iyNd = Nd4j.create(iy);
INDArray izNd = Nd4j.create(iz);
INDArray oxNd = Nd4j.create(ox);
INDArray oyNd = Nd4j.create(oy);
INDArray ozNd = Nd4j.create(oz);
INDArray iNd = Nd4j.hstack(ixNd, iyNd, izNd);
INDArray oNd = Nd4j.hstack(oxNd, oyNd, ozNd);
DataSet dataSet = new DataSet(iNd, oNd);
List<DataSet> listDs = dataSet.asList();
Collections.shuffle(listDs, rng);
return new ListDataSetIterator(listDs, batchSize);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(-1);
return null;
}
}
这里是网络：
// Set up network. 784 in/out (as MNIST images are 28x28).
// 784 -> 250 -> 10 -> 250 -> 784
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).iterations(1)
.weightInit(WeightInit.XAVIER).updater(Updater.ADAGRAD).activation("relu")
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(learningRate)
.regularization(true).l2(0.0001).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(1).build())
.layer(1, new OutputLayer.Builder().nIn(1).nOut(3).lossFunction(LossFunctions.LossFunction.MSE).build())
.pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.setListeners(Collections.singletonList((IterationListener) new ScoreIterationListener(1)));
// Load data and split into training and testing sets. 40000 train,
// 10000 test
DataSetIterator iter = getTrainingData(batchSize, rng);
// Train model:
int nEpochs = 30;
while (iter.hasNext()) {
DataSet ds = iter.next();
for (int epoch = 0; epoch < nEpochs; epoch++) {
net.fit(ds.getFeatures(), ds.getLabels());
System.out.println("Epoch " + epoch + " complete");
}
}
我的错误是：
Exception in thread "main" java.lang.IllegalStateException: Mis matched lengths: [9000] != [3]
at org.nd4j.linalg.util.LinAlgExceptions.assertSameLength(LinAlgExceptions.java:39)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2786)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2767)
at org.nd4j.linalg.api.ndarray.BaseNDArray.sub(BaseNDArray.java:2547)
at org.deeplearning4j.nn.layers.BaseOutputLayer.getGradientsAndDelta(BaseOutputLayer.java:182)
at org.deeplearning4j.nn.layers.BaseOutputLayer.backpropGradient(BaseOutputLayer.java:161)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.calcBackpropGradients(MultiLayerNetwork.java:1125)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.backprop(MultiLayerNetwork.java:1077)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(MultiLayerNetwork.java:1817)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:152)
at org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(StochasticGradientDescent.java:54)
at org.deeplearning4j.optimize.Solver.optimize(Solver.java:51)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1445)
at org.deeplearning4j.examples.feedforward.anomalydetection.IoTAnomalyExample.main(IoTAnomalyExample.java:110)
我很确定我弄乱了训练数据 - 训练数据的形状是 3000 行，3 列 - 与目标相同（完全相同的数据，因为我想构建一个自动编码器） - 测试数据可以在这里找到： https://pmqsimulator-romeokienzler-2310.mybluemix.net/data
有什么想法吗？
感谢Alex Black of Skymind，这就是解决方案（弄错了形状）
INDArray ixNd = Nd4j.create(ix, new int[]{3000,1});
INDArray iyNd = Nd4j.create(iy, new int[]{3000,1});
INDArray izNd = Nd4j.create(iz, new int[]{3000,1});
INDArray oxNd = Nd4j.create(ox, new int[]{3000,1});
INDArray oyNd = Nd4j.create(oy, new int[]{3000,1});
INDArray ozNd = Nd4j.create(oz, new int[]{3000,1});