From dc279fa73083b1fc01bbdf7bbb84975d0875125c Mon Sep 17 00:00:00 2001 From: Zafarali Ahmed Date: Wed, 23 Nov 2016 13:48:05 -0500 Subject: [PATCH 1/2] Fix ESLint Styling issues --- src/feedforwardNeuralNetwork.js | 2 +- src/layer.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/feedforwardNeuralNetwork.js b/src/feedforwardNeuralNetwork.js index b342ec9..0ab80e6 100644 --- a/src/feedforwardNeuralNetwork.js +++ b/src/feedforwardNeuralNetwork.js @@ -39,7 +39,7 @@ class FeedforwardNeuralNetwork { layerOptions = layerOptions || new Array(layersSize.length-1); // output layer must be a sigmoid to give probabilities - layerOptions.push({nonLinearity:'sigmoid'}) + layerOptions.push({nonLinearity: 'sigmoid'}) if(layerOptions.length !== layersSize.length){ throw Error('Must have the same number of layer options as layer size'); diff --git a/src/layer.js b/src/layer.js index bc1b2e8..24f2239 100644 --- a/src/layer.js +++ b/src/layer.js @@ -139,7 +139,7 @@ function sigmoidGradient(value) { * @returns {number} **/ -function tanh(value){ +function tanh(value) { return Math.tanh(value); } @@ -151,8 +151,8 @@ function tanh(value){ * @returns {number} **/ -function tanhGradient(value){ - return 1-Math.pow(value, 2) +function tanhGradient(value) { + return 1 - Math.pow(value, 2) } From bde6cd576a83d9c5c8c749b7fb506d9fafc1e2df Mon Sep 17 00:00:00 2001 From: Zafarali Ahmed Date: Fri, 25 Nov 2016 19:19:39 -0500 Subject: [PATCH 2/2] feat: add tanh non-linearity for layers (#3) * Add tanh non-linearity for layers * Add layerOptions to README, also check to see if no nonlinearity is specified * Fix ESLint Styling issues --- README.md | 1 + src/feedforwardNeuralNetwork.js | 21 ++++- src/layer.js | 49 ++++++++++- test/test.js | 140 +++++++++++++++++++++++++++++++- 4 files changed, 204 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6933168..ce94096 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ found here: __Options__ * `hiddenLayers` - Array with the size of each hidden layer in the FNN. +* `hiddenOptions` - (optional) Array with the options for each layer of the FNN specifiying activating functions of the form `{nonLinearity:'sigmoid' or 'tanh'}`. * `iterations` - Maximum number of iterations of the algorithm. * `learningRate` - The learning rate (number). * `momentum` - The regularization term (number). diff --git a/src/feedforwardNeuralNetwork.js b/src/feedforwardNeuralNetwork.js index 6e3971c..0ab80e6 100644 --- a/src/feedforwardNeuralNetwork.js +++ b/src/feedforwardNeuralNetwork.js @@ -31,15 +31,28 @@ class FeedforwardNeuralNetwork { * Build the Neural Network with an array that represent each hidden layer size. * * @param {Array} layersSize - Array of sizes of each layer. + * @param {Array} layerOptions - Array containing the options for each layer */ - buildNetwork(layersSize) { + buildNetwork(layersSize, layerOptions) { layersSize.push(this.outputSize); + layerOptions = layerOptions || new Array(layersSize.length-1); + + // output layer must be a sigmoid to give probabilities + layerOptions.push({nonLinearity: 'sigmoid'}) + + if(layerOptions.length !== layersSize.length){ + throw Error('Must have the same number of layer options as layer size'); + } + this.layers = new Array(layersSize.length); for (var i = 0; i < layersSize.length; ++i) { var inSize = (i == 0) ? this.inputSize : layersSize[i - 1]; - this.layers[i] = new Layer(inSize, layersSize[i]); + + var options = layerOptions[i] || undefined; + + this.layers[i] = new Layer(inSize, layersSize[i], options); } this.layers[this.layers.length - 1].isSigmoid = false; @@ -114,7 +127,9 @@ class FeedforwardNeuralNetwork { var learningRate = options.learningRate === undefined ? 0.1 : options.learningRate; var momentum = options.momentum === undefined ? 0.1 : options.momentum; - this.buildNetwork(hiddenLayers); + var layerOptions = options.hiddenOptions; + + this.buildNetwork(hiddenLayers, layerOptions); for (var i = 0; i < iterations; ++i) { for (var j = 0; j < predictions.length; ++j) { diff --git a/src/layer.js b/src/layer.js index e7afff6..24f2239 100644 --- a/src/layer.js +++ b/src/layer.js @@ -10,12 +10,25 @@ class Layer { * @param outputSize * @constructor */ - constructor(inputSize, outputSize) { + constructor(inputSize, outputSize, options) { + + options = options || {nonLinearity:'sigmoid'}; + this.output = Matrix.zeros(1, outputSize).getRow(0); this.input = Matrix.zeros(1, inputSize + 1).getRow(0); //+1 for bias term this.deltaWeights = Matrix.zeros(1, (1 + inputSize) * outputSize).getRow(0); this.weights = randomInitializeWeights(this.deltaWeights.length, inputSize, outputSize); - this.isSigmoid = true; + + this.isSigmoid = options.nonLinearity === 'sigmoid'; + this.isTanh = options.nonLinearity === 'tanh'; + + if(!this.isSigmoid && !this.isTanh){ + throw Error('Must define non-linearity as sigmoid or tanh.') + } + // logical XOR, cannot be both at the same time. + if(!((this.isSigmoid || this.isTanh) && !(this.isSigmoid && this.isTanh))){ + throw Error('Cannot have both sigmoid and tanh linearities.'); + } } /** @@ -35,6 +48,8 @@ class Layer { } if (this.isSigmoid) this.output[i] = sigmoid(this.output[i]); + if (this.isTanh) + this.output[i] = tanh(this.output[i]); offs += this.input.length; } @@ -59,6 +74,9 @@ class Layer { if (this.isSigmoid) delta *= sigmoidGradient(this.output[i]); + if (this.isTanh) + delta *= tanhGradient(this.output[i]); + for (var j = 0; j < this.input.length; ++j) { var index = offs + j; nextError[j] += this.weights[index] * delta; @@ -97,7 +115,7 @@ function randomInitializeWeights(numberOfWeights, inputSize, outputSize) { } /** - * Function that calculates the sigmoid (logistic) function. + * Function that calculates the sigmoid (logistic) function at some value * @param value * @returns {number} */ @@ -107,9 +125,34 @@ function sigmoid(value) { /** * Function that calculates the derivate of the sigmoid function. + * given the value of the sigmoid function at that point * @param value * @returns {number} */ function sigmoidGradient(value) { return value * (1 - value); } + +/** + * Function that caclulates the hyperbolic tangent (tanh) function at some value + * @param value + * @returns {number} +**/ + +function tanh(value) { + return Math.tanh(value); +} + +/** + * Function that caclulates the derivative of + * hyperbolic tangent (tanh) function given the value + * of the hyperbolic tangent function at that point + * @param value + * @returns {number} +**/ + +function tanhGradient(value) { + return 1 - Math.pow(value, 2) +} + + diff --git a/test/test.js b/test/test.js index 2a32867..2a0d3d9 100644 --- a/test/test.js +++ b/test/test.js @@ -2,7 +2,7 @@ var FeedforwardNeuralNetwork = require(".."); -describe('Feedforward Neural Networks', function () { +describe('Feedforward Neural Networks using sigmoid nonlinearity', function () { it('Training the neural network with XOR operator', function () { var trainingSet = [[0, 0], [0, 1], [1, 0], [1, 1]]; @@ -111,3 +111,141 @@ describe('Feedforward Neural Networks', function () { result[0][0].should.be.lessThan(result[0][1]); }); }); + + + +describe('Feedforward Neural Networks using Tanh nonlinearity', function () { + + it('Training the neural network with XOR operator', function () { + var trainingSet = [[0, 0], [0, 1], [1, 0], [1, 1]]; + var predictions = [[0], [1], [1], [0]]; + + var xorNN = new FeedforwardNeuralNetwork(trainingSet, predictions); + var options = { + hiddenLayers: [4], + hiddenOptions: [{nonLinearity:'tanh'}], + iterations: 500, + learningRate : 0.3, + momentum: 0.3 + }; + + xorNN.train(options); + var results = xorNN.predict(trainingSet); + + (results[0]).should.be.approximately(predictions[0], 3e-1); + (results[1]).should.be.approximately(predictions[1], 3e-1); + (results[2]).should.be.approximately(predictions[2], 3e-1); + (results[3]).should.be.approximately(predictions[3], 3e-1); + }); + + it('Training the neural network with AND operator', function () { + var trainingSet = [[0, 0], [0, 1], [1, 0], [1, 1]]; + var predictions = [[1, 0], [1, 0], [1, 0], [0, 1]]; + + var andNN = new FeedforwardNeuralNetwork(trainingSet, predictions); + var options = { + hiddenLayers: [3], + hiddenOptions: [{nonLinearity:'tanh'}], + iterations: 500, + learningRate : 0.3, + momentum: 0.3 + }; + andNN.train(options); + + var results = andNN.predict(trainingSet); + + (results[0][0]).should.be.greaterThan(results[0][1]); + (results[1][0]).should.be.greaterThan(results[1][1]); + (results[2][0]).should.be.greaterThan(results[2][1]); + (results[3][0]).should.be.lessThan(results[3][1]); + }); + + it('Export and import', function () { + var trainingSet = [[0, 0], [0, 1], [1, 0], [1, 1]]; + var predictions = [[0], [1], [1], [1]]; + + var orNN = new FeedforwardNeuralNetwork(trainingSet, predictions); + var options = { + hiddenLayers: [4], + hiddenOptions: [{nonLinearity:'tanh'}], + iterations: 500, + learningRate : 0.3, + momentum: 0.3 + }; + orNN.train(options); + + var model = orNN.toJSON(); + var neworNN = FeedforwardNeuralNetwork.load(model); + + var results = neworNN.predict(trainingSet); + + (results[0]).should.be.approximately(predictions[0], 3e-1); + (results[1]).should.be.approximately(predictions[1], 3e-1); + (results[2]).should.be.approximately(predictions[2], 3e-1); + (results[3]).should.be.approximately(predictions[3], 3e-1); + }); + + it('multiclass clasification', function () { + var trainingSet = [[0, 0], [0, 1], [1, 0], [1, 1]]; + var predictions = [[2], [0], [1], [0]]; + + var nn = new FeedforwardNeuralNetwork(trainingSet, predictions); + var options = { + hiddenLayers: [4], + hiddenOptions: [{nonLinearity:'tanh'}], + iterations: 300, + learningRate : 0.5, + momentum: 0.1 + }; + nn.train(options); + + var result = nn.predict(trainingSet); + + result[0].should.be.approximately(2, 1e-1); + result[1].should.be.approximately(0, 1e-1); + result[2].should.be.approximately(1, 1e-1); + result[3].should.be.approximately(0, 1e-1); + }); + + it('big case', function () { + var trainingSet = [[1, 1], [1, 2], [2, 1], [2, 2], [3, 1], [1, 3], [1, 4], [4, 1], + [6, 1], [6, 2], [6, 3], [6, 4], [6, 5], [5, 5], [4, 5], [3, 5]]; + var predictions = [[1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], + [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1]]; + + var nn = new FeedforwardNeuralNetwork(trainingSet, predictions); + var options = { + hiddenLayers: [10], + hiddenOptions: [{nonLinearity:'tanh'}], + iterations: 200, + learningRate : 0.1, + momentum: 0.1 + }; + nn.train(options); + + var result = nn.predict([[5, 4]]); + + result[0][0].should.be.lessThan(result[0][1]); + }); + + it('multi-layer neural network with tanh and sigmoid layers', function () { + var trainingSet = [[1, 1], [1, 2], [2, 1], [2, 2], [3, 1], [1, 3], [1, 4], [4, 1], + [6, 1], [6, 2], [6, 3], [6, 4], [6, 5], [5, 5], [4, 5], [3, 5]]; + var predictions = [[1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], + [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1]]; + + var nn = new FeedforwardNeuralNetwork(trainingSet, predictions); + var options = { + hiddenLayers: [10, 10], + hiddenOptions: [{nonLinearity:'tanh'}, {nonLinearity:'sigmoid'}], + iterations: 200, + learningRate : 0.1, + momentum: 0.1 + }; + nn.train(options); + + var result = nn.predict([[5, 4]]); + + result[0][0].should.be.lessThan(result[0][1]); + }); +}); \ No newline at end of file