Detailed derivation of lstm neural network and realization of C + + __c++

LSTM Hidden Neuron structure:

Detailed structure of lstm hidden neurons:

Let the program itself learn whether to carry, so learn to add #include "iostream" #include "math.h" #include "stdlib.h" #include "time.h" #include "vector" #inc

Lude "Assert.h" using namespace std;
#define INNODE 2//input knot number, will input 2 addends #define 26//hidden knot points, storage "carry Bit" #define OUTNODE 1//output knot points, will output a forecast #define ALPHA 0.1//Learning rate #define BINARY_DIM 8//binary number Maximum length #define RANDVAL (double) rand ()/Rand_max * Hi GH) #define Uniform_plus_minus_one (double) (2.0 * RAND ())/(double Rand_max + 1.0)-1.0)//uniformly randomly distributed int largest_  Number = (POW (2, Binary_dim));

The maximum decimal number//activation function that corresponds to the binary maximum length double sigmoid (double x) {return 1.0/(1.0 + exp (-X));  
The derivative of the activation function, y is the activation function value double dsigmoid (double y) {return y * (1.0-y);
    The derivative of//tanh, Y is the Tanh value double Dtanh (double y) {y = Tanh (y);  
return 1.0-y * y;
    ///Converts a 10-ary integer to a 2-number void int2binary (int n, int *arr) {int i = 0;
        while (n) {arr[i++] = n% 2;
    n/= 2;
while (I < Binary_dim)        arr[i++] = 0;
    Class RNN {public:rnn ();
    Virtual ~rnn ();

void Train ();     Public:double W_i[innode][hidenode];   The weight matrix of the input gate in the connection input and the hidden layer unit double U_i[hidenode][hidenode];     The weighted matrix double w_f[innode][hidenode of the input gate in the upper hidden layer output and the implicit layer unit is connected;   The weight matrix of the forgotten gate in the connection input and the hidden layer unit double U_f[hidenode][hidenode];     The weighted matrix double w_o[innode][hidenode of the forgotten gate in the upper hidden layer and the hidden layer element;   The weight matrix of the forgotten gate in the connection input and the hidden layer unit double U_o[hidenode][hidenode];     The weighted matrix of the hidden layer connected to the upper hidden layer and the present time is double w_g[innode][hidenode];   Weight matrix double U_g[hidenode][hidenode] for generating new memories;  Weight matrix double W_out[hidenode][outnode] for generating new memories;             The weighted matrix of the connection between the hidden layer and the output layer is double *x;     Layer 0 Output value, the input vector directly set//double *layer_1;             Layer 1 output value double *y;

Layer 2 output value};  void Winit (double w[], int n)//weights initialization {for (int i=0; i<n; i++) w[i] = Uniform_plus_minus_one;
    Uniform Random Distribution} rnn::rnn () {x = new Double[innode];
    y = new Double[outnode]; Winit ((double*) w_i, Innode * HIDEnode);
    Winit ((double*) u_i, Hidenode * hidenode);
    Winit ((double*) w_f, Innode * hidenode);
    Winit ((double*) u_f, Hidenode * hidenode);
    Winit ((double*) w_o, Innode * hidenode);
    Winit ((double*) u_o, Hidenode * hidenode);
    Winit ((double*) w_g, Innode * hidenode);
    Winit ((double*) u_g, Hidenode * hidenode);
Winit ((double*) w_out, Hidenode * outnode);
    } rnn::~rnn () {delete x;
Delete y;
    } void Rnn::train () {int epoch, I, J, K, M, p;      Vector<double*> I_vector;      Input door vector<double*> f_vector;      Forgotten Door vector<double*> O_vector;      Output Door vector<double*> g_vector;      New Memory vector<double*> S_vector;      State value vector<double*> h_vector;        Output value vector<double> Y_delta;  Save error on the offset of the output layer for (epoch=0; epoch<11000; epoch++)//training times {double e = 0.0;               Error int Predict[binary_dim];
Save the predicted values for each generation memset (predict, 0, sizeof (predict));
        int a_int = (int) randval (largest_number/2.0);
        Randomly generating a addends a int a[binary_dim];                 Int2binary (A_int, a);  Convert to binary number int b_int = (int) randval (largest_number/2.0);
        Randomly generating another addends b int B[binary_dim];                 Int2binary (B_int, b);            Convert to binary number int c_int = A_int + b_int;
        True and C int C[binary_dim];                 Int2binary (C_int, C);     To binary number//at 0 The time is no previous hidden layer, so initialize a full 0 double *s = new Double[hidenode];     state value Double *h = new Double[hidenode];
            Output value for (i=0; i