Neural Network Learning

class Neuron {

  constructor(value, connections, bias, layer) {

    this.value = value;

    this.connections = connections; // Array of {"neuron": Neuron, "weight": Float}

    this.bias = bias;

    this.layer = layer;

    this.error = 0;

  }

  propagate() {

    for (var i = 0; i < this.connections.length; i++) {

      this.connections[i].neuron.value += this.value * this.connections[i].weight;

    }

  }

}

class Network {

  constructor(layerData) {

    this.layers = []; // layerData is an array of arrays of values. [ [ Float, Float, ... ], ... ]

    this.learningRate = 0.1;

    // Initialize neurons with random connection weights

    var layer, connections, neuronIndex, nextNeuronIndex;

    for (layer = layerData.length - 1; layer > -1; layer--) {

      this.layers.unshift([]);

      for (neuronIndex = 0; neuronIndex < layerData[layer].length; neuronIndex++) {

        connections = [];

        if (layer != layerData.length - 1) {

          for (nextNeuronIndex = 0; nextNeuronIndex < this.layers[1].length; nextNeuronIndex++) {

            connections.push({"neuron": this.layers[1][nextNeuronIndex], "weight": Math.random()*2 - 1});

          }

        }

        this.layers[0].push(new Neuron(layerData[layer][neuronIndex], connections, 0, layer));

      }

    }

  }

  display() {

    var layer, neuronIndex, layerNum = this.layers.length, neuronNum, connectionIndex, connectionNum, connection,

        neuronXSpace = windowWidth / 15, neuronYSpace = windowHeight / 10, neuronSize = windowHeight / 16;

    textSize(neuronSize * 0.3);

    for (layer = 0; layer < layerNum; layer++) {

      neuronNum = this.layers[layer].length;

      for (neuronIndex = 0; neuronIndex < neuronNum; neuronIndex++) {

        // connections

        for (connectionIndex = 0; connectionIndex < this.layers[layer][neuronIndex].connections.length; connectionIndex++) {

          connection = this.layers[layer][neuronIndex].connections[connectionIndex];

          connectionNum = this.layers[layer][neuronIndex].connections.length;

          stroke(-connection.weight * 200, connection.weight * 200, 0);

          line((layer - layerNum/2) * neuronXSpace     , (neuronIndex - neuronNum/2) * neuronYSpace,

              ((layer + 1) - layerNum/2) * neuronXSpace, (connectionIndex - connectionNum/2) * neuronYSpace);

        }

        // neurons

        noStroke();

        fill(255, 255, 255);

        ellipse((layer - layerNum/2) * neuronXSpace, (neuronIndex - neuronNum/2) * neuronYSpace, neuronSize, neuronSize);

        stroke(0, 0, 0);

        fill(0, 0, 0);

        text(Math.floor(this.layers[layer][neuronIndex].value * 100) / 100,

             (layer - layerNum/2) * neuronXSpace, (neuronIndex - neuronNum/2) * neuronYSpace);

      }

    }

  }

  output(inputValues) {

    // Reset neuron values

    for (layer = 0; layer < this.layers.length; layer++) {

      for (i = 0; i < this.layers[layer].length; i++) {

        this.layers[layer][i].value = 0;

      }

    }

    // Set first layer and propagate without sigmoid

    var i, layer, output = [];

    for (i = 0; i < this.layers[0].length; i++) {

      this.layers[0][i].value = inputValues[i];

      this.layers[0][i].propagate();

    }

    // Propagate after sigmoiding neurons' values to squishify their arbitrary values to between 0 and 1

    for (layer = 1; layer < this.layers.length; layer++) {

      for (i = 0; i < this.layers[layer].length; i++) {

        this.layers[layer][i].value += this.layers[layer][i].bias;

        this.layers[layer][i].value = sigmoid(this.layers[layer][i].value);

        this.layers[layer][i].propagate();

      }

    }

    // Convert output Neuron list to Float list

    for (i = 0; i < this.layers[this.layers.length - 1].length; i++) {

      output.push(this.layers[this.layers.length - 1][i].value);

    }

    return output;

  }

  loss(trainingData) { // data is an array of {"input": array of Floats, "expectedOutput": array of Floats}

    var netLoss = 0, trainingIndex, output, expected, i;

    for (trainingIndex = 0; trainingIndex < trainingData.length; trainingIndex++) {

      output = this.output(trainingData[trainingIndex].input);

      expected = trainingData[trainingIndex].expectedOutput;

      for (i = 0; i < output.length; i++) {

        netLoss += Math.pow(output[i] - expected[i], 2);

      }

    }

    return netLoss / trainingData.length;

  }

  train(trainingData) {

    for (var example = 0; example < trainingData.length; example++) {

      // Gradient descent optimization by back propagating through the network, shifting weights.

      // New neural network with neuron values as their error. Starts as a clone of this net with the final layer as the loss.

      var output, expected, sigmoidPrimeValue, i;

      output = this.output(trainingData[example].input);

      expected = trainingData[example].expectedOutput;

      for (i = 0; i < output.length; i++) {

        this.layers[this.layers.length - 1][i].error = output[i] - expected[i];

      }

      // Back propagate error by calculating for each layer's neurons the sum of the errors of the connected next layer's neurons

      // * the weight of the connection to that neuron, all * sigmoidPrime(neuron value)

      var layerNum, layer, neuron, neuronIndex, connectionIndex;

      for (layerNum = this.layers.length - 2; layerNum > -1; layerNum--) {

        layer = this.layers[layerNum];

        for (neuronIndex = 0; neuronIndex < layer.length; neuronIndex++) {

          neuron = layer[neuronIndex];

          neuron.error = 0;

          sigmoidPrimeValue = sigmoidPrime(neuron.value);

          for (connectionIndex = 0; connectionIndex < neuron.connections.length; connectionIndex++) {

            neuron.error += neuron.connections[connectionIndex].weight * neuron.connections[connectionIndex].neuron.error;

          }

          neuron.error *= sigmoidPrimeValue;

        }

      }

​

      // We know the cost of every neuron, its contribution to the network's error. Neuron weights can now be optimized with Gradient Descent.

      var delta, connection;

      for (layerNum = 0; layerNum < this.layers.length - 1; layerNum++) {

        layer = this.layers[layerNum];

        for (neuronIndex = 0; neuronIndex < layer.length; neuronIndex++) {

          neuron = layer[neuronIndex];

          for (connectionIndex = 0; connectionIndex < neuron.connections.length; connectionIndex++) {

            connection = neuron.connections[connectionIndex];

            delta = neuron.value * connection.neuron.error * connection.neuron.value * (1 - connection.neuron.value);

            connection.weight -= delta * this.learningRate;

          }

        }

      }

    }

  }

}

​

function sigmoid(x) {

  return 1 / (1 + Math.exp(-x));

}

function sigmoidPrime(x) {

  var expX = Math.exp(x);

  return expX / Math.pow(1 + expX, 2);

}

​

var layerData = [], neuronsPerLayer = [1, 5, 5, 1]; // Elements represent how many neurons per layer.

for (var layer = 0, neuronIndex; layer < neuronsPerLayer.length; layer++) {

  layerData.push([]);

  for (neuronIndex = 0; neuronIndex < neuronsPerLayer[layer]; neuronIndex++) {

    layerData[layerData.length - 1].push(0); // Initial value.

  }

}

var network = new Network(layerData);

​

var trainingData = [], input;

for (var i = 0; i < 1000; i++) {

  input = Math.random(); // Train the network to apply a function to its input.

  trainingData.push({"input": [input], "expectedOutput": [Math.sin(input)]});

}

​

function setup() {

  createCanvas(windowWidth, windowHeight);

  textAlign(CENTER, CENTER);

}

​

function draw() {

  translate(windowWidth/2, windowHeight/2);

  background(0, 0, 0);

  if (millis() < 2000)

    network.train(trainingData);

  network.display();

  fill(255, 255, 255);

  textSize(windowHeight / 25);

  text("Loss: " + Math.floor(network.loss(trainingData) * 1000000) / 1000000, -0.4 * windowWidth, -0.4 * windowHeight);

}