I'm trying to make a neural network using MATLAB. But I'm finding that many problems do not get solved.
These are the formulas on which I based: $$ Layers\ in\ Neural\ Network = [0, ..., i, ..., out] $$ $$ W_{i,j} = weights\ of\ the\ layer_i\ to\ layer_j\ \in M_{m\ x \ n}\\ B_{i,j} = weights\ of\ the\ layer_i\ to\ layer_j \in \mathbb{R}^n \\ d_{i} = delta\ correction\ of\ the\ layer_i\ \in \mathbb{R}^n \\ out_{j} = output\ signal\ of\ the\ layer_j\ \in \mathbb{R}^n \\ $$
Update Weights Output layer $$ d_{output} = out_{output}·(1-out_{output})·(out_{output} - Y)\\ W_{j, output}^{+} = W_{j,output} - n·out_{j}^{T}·d_{output} \\ B_{j, output}^{+} = B_{j,output} + n·d_{output} \\ $$
Update Weights Hidden layers $$ d_{i} = out_{i}·(1-out_{i})·(d_{i+1}·W_{i+1, i+2}^T)\\ W_{i,j}^{+} = W_{i,j} - n·out_{i-1}^{T}·d_{i} \\ B_{i,j}^{+} = B_{i,j} + n·d_{i} \\ $$
Update Weights First Hidden layer $$ d_{1} = out_{1}·(1-out_{1})·(d_{2}·W_{2,3}^T)\\ W_{1,2}^{+} = W_{1,2} - n·X^T·d_{1} \\ B_{1,2}^{+} = B_{1,2} + n·d_{1} \\ $$
This is the code I generated:
classdef NeuralNetwork < handle
properties
% Wjk j inputs to k outputs (j inputs/neuron)
LW;
% Bk (k bias)
LB;
num_inputs;
num_outputs;
n_descensParameter;
end
methods
function obj = NeuralNetwork(num_inputs, num_outputs, hidden_layers, n_descensParameter)
obj.num_inputs = num_inputs;
obj.num_outputs = num_outputs;
obj.n_descensParameter = n_descensParameter;
size_ant = obj.num_inputs;
for i=1:length(hidden_layers)
obj.LW{i} = zeros(size_ant, hidden_layers(i));
obj.LB{i} = zeros(1, hidden_layers(i));
size_ant = hidden_layers(i);
end
obj.LW{length(hidden_layers)+1} = zeros(hidden_layers(end), num_outputs);
obj.LB{length(hidden_layers)+1} = zeros(1, num_outputs);
end
function InitWeights(obj)
b = sqrt(6)/sqrt(size(obj.LW{1}, 2)+obj.num_inputs);
obj.LW{1} = -b + (2*b).*rand(size(obj.LW{1}));
for i=2:length(obj.LW),
obj.LW{i} = -b + (2*b).*rand(size(obj.LW{i}));
b = sqrt(6)/sqrt(size(obj.LW{i}, 2)+size(obj.LW{i-1}, 2));
end
end
function out = predict(obj, X)
inputs = X;
for i=1:length(obj.LW)
inputs = obj.activationNeuronFunc(inputs*obj.LW{i} + obj.LB{i});
end
out = obj.cOutput(inputs);
end
function d = activationNeuronFunc(obj, x)
d = (1./(1+exp(-x)));
end
function d = cOutput(obj, o)
d = round(o); % softmax
end
function d = doutput(obj, o)
d = o.*(1-o);
end
function rval = cicle_backpropagation(obj, X, Y)
o = {};
inputs = X;
for i=1:length(obj.LW)
o{i} = obj.activationNeuronFunc(inputs*obj.LW{i} + obj.LB{i});
inputs = o{i};
end
LWplus = cell(size(obj.LW));
d = cell(size(obj.LW));
d{end} = obj.doutput(o{end}).*(o{end}-Y);
LWplus{end} = obj.LW{end} - obj.n_descensParameter.*(transpose(o{end-1})*d{end});
obj.LB{end} = obj.LB{end} + obj.n_descensParameter.*d{end};
for i=length(obj.LW)-1:-1:2
d{i} = obj.doutput(o{i}).*(d{i+1}*transpose(obj.LW{i+1}));
LWplus{i} = obj.LW{i} - obj.n_descensParameter.*(transpose(o{i-1})*d{i});
obj.LB{i} = obj.LB{i} + obj.n_descensParameter.*d{i};
end
d{1} = obj.doutput(o{1}).*(d{2}*transpose(obj.LW{2}));
LWplus{1} = obj.LW{1} - obj.n_descensParameter.*(transpose(X)*d{1});
obj.LB{1} = obj.LB{1} + obj.n_descensParameter.*d{1};
obj.LW = LWplus;
rval = sum(abs(Y-o{end}));
end
end
end
And this is a small test which should create the XOR expression:
% Learn
a = NeuralNetwork(2,2,[2], 1);
a.InitWeights()
X = [1, 1; 1, 0; 0, 1; 0, 0];
Y = [0, 1; 1, 0; 1, 0; 0, 1];
for i=1:1500,
for j=1:4,
a.cicle_backpropagation(c(j,:), b(j,:))
end
end
% Validate
a.predict([1,1]) % Must be 0, 1
a.predict([1,0]) % Must be 1, 0
a.predict([0,1]) % Must be 1, 0
a.predict([0,0]) % Must be 0, 1
The problem is that it does not learn correctly, I feel that there is an error in the code or formulas that have deduced. Could correct me? I'm pretty rookie and I want to learn to program it to understand fully.