为什么我不能用 C++ 构建一个没有依赖关系的神经网络，即使它可以在 Numpy 中工作？

问题描述 投票：0回答：1

我有一个小型 python 程序，它使用 numpy 在 Python 中构建一个神经网络，另一个使用 C++ 构建神经网络。 Numpy 版本的学习速度很快，而且通常方向正确，但 C++ 的准确率从未超过 20%。它通常会上下振荡，有时会朝正确的方向大幅跳跃。

data = pd.read_csv('data.csv', header=None)
data = np.array(data)

m, n = data.shape
#np.random.shuffle(data) # shuffle before splitting into dev and training sets

#data_dev = data[0:1000].T
#Y_dev = data_dev[0]
#X_dev = data_dev[1:n]
#X_dev = X_dev / 255.

data_train = data.T
Y_train = data_train[0]
X_train = data_train[1:n]
X_train = X_train /255


_,m_train = X_train.shape

def get_predictions(A2):
    return np.argmax(A2, 0)

def get_accuracy(predictions, Y):
    return np.sum(predictions == Y) / Y.size
def one_hot(Y):
    one_hot_Y = np.zeros((Y.size, Y.max() + 1))
    one_hot_Y[np.arange(Y.size), Y] = 1
    one_hot_Y = one_hot_Y.T
    return one_hot_Y
def init_params(n_classes, n_features, n_hidden):
    W1 = np.random.rand(n_hidden, n_features) - 0.5
    b1 = np.random.rand(n_hidden, 1) - 0.5
    W2 = np.random.rand(n_classes, n_hidden) - 0.5
    b2 = np.random.rand(n_classes, 1) - 0.5
    return W1, b1, W2, b2
def init_params(n_classes, n_features, n_hidden, precise=False):
    W1 = np.random.rand(n_hidden, n_features) - 0.5
    b1 = np.random.rand(n_hidden, 1) - 0.5
    W2 = np.random.rand(n_classes, n_hidden) - 0.5
    b2 = np.random.rand(n_classes, 1) - 0.5
    if precise:
        return  W1, b1, W2, b2
    return  W1.astype(np.float32), b1.astype(np.float32), W2.astype(np.float32), b2.astype(np.float32) 
def init_params_non_random(n_classes, n_features, n_hidden, precise = False):
    W1 = np.zeros( (n_hidden, n_features)) + .1
    
    b1 = np.zeros( (n_hidden, 1) ) + .1
    W2 = np.zeros( (n_classes, n_hidden) ) + .1
    b2 = np.zeros( (n_classes, 1)) + .1
    if precise:
        return  W1, b1, W2, b2
    return  W1.astype(np.float32), b1.astype(np.float32), W2.astype(np.float32), b2.astype(np.float32) 
    
def ReLU(Z):
    return np.maximum(Z, 0)

def softmax(Z):
    A = np.exp(Z) / sum(np.exp(Z))
    return A
    
def ReLU_deriv(Z):
    return Z > 0
    
W1, b1, W2, b2 = init_params(10,784, 10)
alpha = .9
one_hot_y = one_hot(Y_train)

num_iterations = 1
i = 0

W1, b1, W2, b2 = init_params(10,784, 10, True)

i = 0
while i <10: # Forward
    i+=1
    Z1 = W1.dot(X_train) + b1
    A1 = ReLU(Z1)
    Z2 = W2.dot(A1) + b2
    A2 = softmax(Z2)
    
    dZ2 = A2 - one_hot_y ### absolute difference between the prediction and the target. 
    db2 = 1 / m * np.sum(dZ2)
    dW2 = 1 / m * dZ2.dot(A1.T)
    dZ1 = W2.T.dot(dZ2) * ReLU_deriv(Z1)
    dW1 = 1 / m * dZ1.dot(X_train.T)
    db1 = 1 / m * np.sum(dZ1)
    
    W1 = W1 - alpha * dW1
    b1 = b1 - alpha * db1    
    #W2 = W2 - alpha * dW2  
    b2 = b2 - alpha * db2    
    predictions = get_predictions(A2)
    print(get_accuracy(predictions, Y_train))

输出：

0.07441666666666667
0.11155
0.15333333333333332
0.16476666666666667
0.20651666666666665
0.23701666666666665
0.25858333333333333
0.27908333333333335
0.29013333333333335
0.3076833333333333

C++版本：

#include <bits/stdc++.h>
#include <strings.h>

#include <cmath>
#include <csignal>
#include <fstream>
#include <iostream>
#include <ostream>
#include <random>
#include <sstream>
#include <string>

using namespace std;

#define ROW 60000
#define COL 785
#define FEAT COL - 1
#define CLASSES 10
#define HIDDEN 10
#define MAX 60000
#define NORM 255  /// max value for normalization
const double euler = 2.71828182845904523536;

typedef float precision;
precision (*x_train)[ROW] = new precision[FEAT][ROW];
int *y = new int[ROW];

precision w1[HIDDEN][FEAT];
precision w2[CLASSES][HIDDEN];
precision w2T[HIDDEN][CLASSES];
precision b1[HIDDEN];
precision b2[CLASSES];
precision (*z1)[ROW] = new precision[HIDDEN][ROW];
precision (*a1)[ROW] = new precision[HIDDEN][ROW];

precision (*z2)[ROW] = new precision[CLASSES][ROW];
precision (*a2)[ROW] = new precision[CLASSES][ROW];

precision (*dz1)[ROW] = new precision[HIDDEN][ROW];
precision (*dz2)[ROW] = new precision[CLASSES][ROW];

precision dw2[CLASSES][HIDDEN];

precision db2;
precision db1;

precision (*dw1)[FEAT] = new precision[HIDDEN][FEAT];
int (*one_hot_y)[ROW] = new int[CLASSES][ROW];

int n_correct = 0;  // the number of correct predictions
precision alpha = 0.9;

int main() {
  std::ifstream file("data.csv");
  if (!file.is_open()) {
    std::cerr << "Error opening file data.csv" << endl;
  }
  std::string line;
  int row = 0;
  int col = 0;
  while (std::getline(file, line) && row < ROW) {
    std::istringstream iss(line);

    string s;
    while (getline(iss, s, ',') && col < COL) {
      if (col == 0) {
        y[row] = stoi(s);
      } else {
        x_train[col - 1][row] = stod(s) / (NORM);
      }
      col++;
    }
    col = 0;
    row++;
  }

  file.close();
  // one hot encoding
  //
  for (int i = 0; i < ROW; i++) {
    int index = y[i];
    for (int j = 0; j < CLASSES; j++) {
      one_hot_y[j][i] = (j == index) ? 1 : 0;
    }
  }

  // populate the weights;
  //
  std::random_device rd;
  std::mt19937 gen(rd());
  std::uniform_real_distribution<precision> dis(0, 1);

  bool test = false;
  for (int i = 0; i < HIDDEN; i++) {
    for (int j = 0; j < FEAT; j++) {
      w1[i][j] = test ? 0.1 : dis(gen) - .5;
      /// ;
    }
  }

  for (int i = 0; i < CLASSES; i++) {
    for (int j = 0; j < HIDDEN; j++) {
      w2[i][j] = test ? 0.1 : dis(gen) - .5;
    }
  }

  for (int i = 0; i < HIDDEN; i++) {
    b1[i] = test ? 0.1 : dis(gen) - .5;
  }
  for (int i = 0; i < CLASSES; i++) {
    b2[i] = test ? 0.1 : dis(gen) - .5;
  }

  while (n_correct / (precision)ROW < .95) {
    // Z1 = W1.dot(X_train) + b1
    // A1 = RelU(Z1)
    //

    for (int i = 0; i < HIDDEN; i++) {
      for (int j = 0; j < ROW; j++) {
        z1[i][j] = 0.0;
        for (int k = 0; k < FEAT; k++) {
          z1[i][j] += w1[i][k] * x_train[k][j];
        }
        z1[i][j] += b1[i];
        a1[i][j] = (z1[i][j] <= 0.0) ? 0.0 : z1[i][j];
      }
    }

    //
    // Z2 = W2.dot(A1) + b2
    //
    for (int i = 0; i < CLASSES; i++) {
      for (int j = 0; j < ROW; j++) {
        z2[i][j] = 0.0;  // initialize z2
        for (int k = 0; k < HIDDEN; k++) {
          z2[i][j] += (w2[i][k] * a1[k][j]);
        }
        z2[i][j] += b2[i];
      }
    }

    int n_correct = 0;
    for (int i = 0; i < ROW; i++) {
      precision exp_sum = 0.0;
      for (int j = 0; j < CLASSES; j++) {
        a2[j][i] = pow(euler, z2[j][i]);

        exp_sum += a2[j][i];
      }
      int prediction = 0;
      precision max = 0;
      for (int j = 0; j < CLASSES; j++) {
        a2[j][i] /= exp_sum;
        if (a2[j][i] > max) {
          prediction += 1;
          max = (a2[j][i]);
        }
      }

      if (y[i] == prediction) {
        n_correct += 1;
      }
    }

    db2 = 0.0;
    for (int i = 0; i < ROW; i++) {
      for (int j = 0; j < CLASSES; j++) {
        dz2[j][i] = a2[j][i] - one_hot_y[j][i];
        db2 += dz2[j][i];
      }
    }
    db2 /= MAX;

    for (int i = 0; i < CLASSES; i++) {
      for (int j = 0; j < HIDDEN; j++) {
        dw2[i][j] = 0.0;
        for (int k = 0; k < ROW; k++) {
          dw2[i][j] += (dz2[i][k] * a1[j][k]);
        }
        dw2[i][j] /= MAX;
      }
    }

    db1 = 0.0;
    // dZ1 = W2.T.dot(dZ2) * ReLU_deriv(Z1)
    //// this is incorrect
    ///
    ///
    ///
    ///
    for (int i = 0; i < CLASSES; i++) {
      for (int j = 0; j < HIDDEN; j++) {
        w2T[j][i] = w2[i][j];
      }
    }
    for (int i = 0; i < HIDDEN; i++) {
      for (int j = 0; j < ROW; j++) {
        dz1[i][j] = 0.0;

        for (int k = 0; k < CLASSES; k++) {
          dz1[i][j] += (w2T[i][k] * dz2[k][j]);
        }

        dz1[i][j] *= (z1[i][j] <= 0 ? 0 : 1);

        db1 += dz1[i][j];
      }
    }

    db1 /= MAX;
    for (int i = 0; i < HIDDEN; i++) {
      for (int j = 0; j < FEAT; j++) {
        dw1[i][j] = 0.0;
        for (int k = 0; k < ROW; k++) {
          dw1[i][j] += (dz1[i][k] * x_train[j][k]);
        }
        dw1[i][j] /= MAX;
      }
    }
    // dw2 appears to be correct, but not dW1
    // dw1 is incorrect because dz1 is incorrect
    // dz1 is based on dz2 and w2. dz2 appears to be correct. w2 must be
    // correct.
    // cout << w2[0][9] << endl;
    for (int i = 0; i < CLASSES; i++) {
      b2[i] -= (alpha * db2);
      for (int j = 0; j < HIDDEN; j++) {
        w2[i][j] -= (alpha * dw2[i][j]);
      }
    }

    for (int i = 0; i < HIDDEN; i++) {
      b1[i] -= (alpha * db1);
      for (int j = 0; j < FEAT; j++) {
        w1[i][j] -= (alpha * dw1[i][j]);
      }
    }

    cout << "Accuracy is " << n_correct / (double)ROW << endl;
  }
  return 0;
}

输出：

Accuracy is 0.0919833
Accuracy is 0.109233
Accuracy is 0.1223
Accuracy is 0.131617
Accuracy is 0.142067
Accuracy is 0.148883
Accuracy is 0.145183
Accuracy is 0.13635
Accuracy is 0.122533
Accuracy is 0.108967
Accuracy is 0.100317

我的结论是，要么我在雅可比矩阵中犯了一个错误，要么在 C++ 实现中存在某种精度/数值稳定性问题。

我正在使用 

-0fast
，但也尝试了其他优化级别以及从 float 到 long double 的所有精度级别。

此外我尝试了不同的权重初始化。

我使用 test 布尔标志来生成可预测的权重，以便我可以将它们与 numpy 版本进行比较。 dz1 的雅可比行列似乎不正确（与 numpy 生成的不同），但我不明白为什么。

c++ numpy neural-network ml
1个回答
0
投票

看起来您正在使用 softmax 函数作为外层，但逻辑有些不正确。

int prediction = 0;
precision max = a2[0][i];
for (int j = 1; j < CLASSES; j++) {
    if (a2[j][i] > max) {
        max = a2[j][i];
        prediction = j;
    }
}

尝试一下可能会有所帮助

最新问题
© www.soinside.com 2019 - 2024. All rights reserved.