[libfann] 203/242: Added momentum parameter
Christian Kastner
chrisk-guest at moszumanska.debian.org
Sat Oct 4 21:10:44 UTC 2014
This is an automated email from the git hooks/post-receive script.
chrisk-guest pushed a commit to tag Version2_0_0
in repository libfann.
commit b2a065594249a561932b92f5e20fccad0f41b52b
Author: Steffen Nissen <lukesky at diku.dk>
Date: Wed Oct 26 13:43:57 2005 +0000
Added momentum parameter
---
benchmarks/.cvsignore | 1 +
benchmarks/benchmark.sh | 42 ++++++++++++++++++---------------------
benchmarks/gnuplot | 22 +++++++++++++++++++-
benchmarks/quality.cc | 12 ++++++++++-
examples/{robot.c => momentums.c} | 38 ++++++++++++++---------------------
examples/robot.c | 2 +-
src/fann.c | 6 +++++-
src/fann_train.c | 29 +++++++++++++++++++++++----
src/include/fann_data.h | 13 +++++++++++-
src/include/fann_train.h | 30 ++++++++++++++++++++++++++++
10 files changed, 140 insertions(+), 55 deletions(-)
diff --git a/benchmarks/.cvsignore b/benchmarks/.cvsignore
index 9c9140f..8b53c3a 100644
--- a/benchmarks/.cvsignore
+++ b/benchmarks/.cvsignore
@@ -6,3 +6,4 @@ performance
quality
shuffle
two-spirals
+allplots.ps
diff --git a/benchmarks/benchmark.sh b/benchmarks/benchmark.sh
index c3e4992..b50fc42 100755
--- a/benchmarks/benchmark.sh
+++ b/benchmarks/benchmark.sh
@@ -12,32 +12,28 @@ function benchmark_algorithm() {
function benchmark_problem() {
#rm -f *_fixed.net
- algo="fann_cascade"; benchmark_algorithm;
- algo="fann_rprop"; benchmark_algorithm;
+# algo="fann_cascade"; benchmark_algorithm;
+# algo="fann_rprop"; benchmark_algorithm;
#./quality_fixed $prob.$algo.train.out_fixed_train $prob.$algo.train.out_fixed_test $prob.$algo.fixed_train.out $prob.$algo.fixed_test.out *_fixed.net
-
-
- algo="fann_rprop_stepwise"; benchmark_algorithm;
- algo="fann_quickprop"; benchmark_algorithm;
- algo="fann_quickprop_stepwise"; benchmark_algorithm;
- algo="fann_batch"; benchmark_algorithm;
- algo="fann_batch_stepwise"; benchmark_algorithm;
+
+# algo="fann_rprop_stepwise"; benchmark_algorithm;
+# algo="fann_quickprop"; benchmark_algorithm;
+ #algo="fann_quickprop_stepwise"; benchmark_algorithm;
+# algo="fann_batch"; benchmark_algorithm;
+ #algo="fann_batch_stepwise"; benchmark_algorithm;
algo="fann_incremental"; benchmark_algorithm;
- algo="fann_incremental_stepwise"; benchmark_algorithm;
+ algo="fann_incremental_momentum"; benchmark_algorithm;
+ #algo="fann_incremental_stepwise"; benchmark_algorithm;
#comment out two following lines if the libraries are not available
- algo="lwnn"; benchmark_algorithm;
- algo="jneural"; benchmark_algorithm;
+# algo="lwnn"; benchmark_algorithm;
+# algo="jneural"; benchmark_algorithm;
}
#comment out some of the lines below if some of the problems should not be benchmarked
-#prob="two-spiral"; n1=20; n2=0; sec_train=20;
-#benchmark_problem;
-#exit;
-
-#prob="building"; n1=16; n2=0; sec_train=$max_seconds_training;
-#benchmark_problem;
+prob="building"; n1=16; n2=0; sec_train=$max_seconds_training;
+benchmark_problem;
prob="diabetes"; n1=4; n2=0; sec_train=$max_seconds_training;
benchmark_problem;
@@ -69,10 +65,10 @@ benchmark_problem;
prob="two-spiral"; n1=20; n2=10; sec_train=$max_seconds_training;
benchmark_problem;
-./performance fann fann_performance.out 1 2048 2 20
-./performance fann_stepwise fann_stepwise_performance.out 1 2048 2 20
-./performance_fixed fann fann_fixed_performance.out 1 2048 2 20
-./performance lwnn lwnn_performance.out 1 2048 2 20
-./performance jneural jneural_performance.out 1 256 2 20
+#./performance fann fann_performance.out 1 2048 2 20
+#./performance fann_stepwise fann_stepwise_performance.out 1 2048 2 20
+#./performance_fixed fann fann_fixed_performance.out 1 2048 2 20
+#./performance lwnn lwnn_performance.out 1 2048 2 20
+#./performance jneural jneural_performance.out 1 256 2 20
gnuplot < gnuplot
diff --git a/benchmarks/gnuplot b/benchmarks/gnuplot
index 5d1f1d0..652ad07 100644
--- a/benchmarks/gnuplot
+++ b/benchmarks/gnuplot
@@ -29,6 +29,7 @@ plot "building.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"building.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"building.fann_batch.test.out" title "fann batch test" with lines 4, \
"building.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "building.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"building.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"building.jneural.test.out" title "jneural incremental test" with lines 9, \
"building.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -37,6 +38,7 @@ plot "building.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"building.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"building.fann_batch.train.out" title "fann batch train" with lines 4, \
"building.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "building.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"building.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"building.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -50,6 +52,7 @@ plot "diabetes.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"diabetes.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"diabetes.fann_batch.test.out" title "fann batch test" with lines 4, \
"diabetes.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "diabetes.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"diabetes.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"diabetes.jneural.test.out" title "jneural incremental test" with lines 9, \
"diabetes.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -58,6 +61,7 @@ plot "diabetes.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"diabetes.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"diabetes.fann_batch.train.out" title "fann batch train" with lines 4, \
"diabetes.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "diabetes.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"diabetes.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"diabetes.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -71,6 +75,7 @@ plot "gene.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"gene.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"gene.fann_batch.test.out" title "fann batch test" with lines 4, \
"gene.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "gene.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"gene.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"gene.jneural.test.out" title "jneural incremental test" with lines 9, \
"gene.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -79,6 +84,7 @@ plot "gene.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"gene.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"gene.fann_batch.train.out" title "fann batch train" with lines 4, \
"gene.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "gene.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"gene.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"gene.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -92,6 +98,7 @@ plot "mushroom.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"mushroom.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"mushroom.fann_batch.test.out" title "fann batch test" with lines 4, \
"mushroom.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "mushroom.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"mushroom.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"mushroom.jneural.test.out" title "jneural incremental test" with lines 9, \
"mushroom.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -100,6 +107,7 @@ plot "mushroom.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"mushroom.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"mushroom.fann_batch.train.out" title "fann batch train" with lines 4, \
"mushroom.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "mushroom.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"mushroom.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"mushroom.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -113,6 +121,7 @@ plot "parity8.fann_rprop.train.out" title "fann rprop train" with lines -1, \
"parity8.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"parity8.fann_batch.train.out" title "fann batch train" with lines 4, \
"parity8.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "parity8.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"parity8.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"parity8.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -126,6 +135,7 @@ plot "parity13.fann_rprop.train.out" title "fann rprop train" with lines -1, \
"parity13.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"parity13.fann_batch.train.out" title "fann batch train" with lines 4, \
"parity13.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "parity13.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"parity13.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"parity13.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -139,6 +149,7 @@ plot "pumadyn-32fm.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"pumadyn-32fm.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"pumadyn-32fm.fann_batch.test.out" title "fann batch test" with lines 4, \
"pumadyn-32fm.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "pumadyn-32fm.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"pumadyn-32fm.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"pumadyn-32fm.jneural.test.out" title "jneural incremental test" with lines 9, \
"pumadyn-32fm.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -147,6 +158,7 @@ plot "pumadyn-32fm.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"pumadyn-32fm.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"pumadyn-32fm.fann_batch.train.out" title "fann batch train" with lines 4, \
"pumadyn-32fm.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "pumadyn-32fm.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"pumadyn-32fm.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"pumadyn-32fm.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -160,6 +172,7 @@ plot "robot.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"robot.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"robot.fann_batch.test.out" title "fann batch test" with lines 4, \
"robot.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "robot.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"robot.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"robot.jneural.test.out" title "jneural incremental test" with lines 9, \
"robot.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -168,6 +181,7 @@ plot "robot.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"robot.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"robot.fann_batch.train.out" title "fann batch train" with lines 4, \
"robot.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "robot.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"robot.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"robot.jneural.train.out" title "jneural incremental train" with lines 9
@@ -180,6 +194,7 @@ plot "soybean.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"soybean.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"soybean.fann_batch.test.out" title "fann batch test" with lines 4, \
"soybean.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "soybean.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"soybean.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"soybean.jneural.test.out" title "jneural incremental test" with lines 9, \
"soybean.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -188,6 +203,7 @@ plot "soybean.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"soybean.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"soybean.fann_batch.train.out" title "fann batch train" with lines 4, \
"soybean.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "soybean.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"soybean.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"soybean.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
@@ -201,6 +217,7 @@ plot "thyroid.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"thyroid.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"thyroid.fann_batch.test.out" title "fann batch test" with lines 4, \
"thyroid.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "thyroid.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"thyroid.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"thyroid.jneural.test.out" title "jneural incremental test" with lines 9, \
"thyroid.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -209,10 +226,11 @@ plot "thyroid.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"thyroid.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"thyroid.fann_batch.train.out" title "fann batch train" with lines 4, \
"thyroid.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "thyroid.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"thyroid.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"thyroid.jneural.train.out" title "jneural incremental train" with lines 9
-#set logscale x
+set logscale x
#set output "two-spiral.ps"
set title "two-spiral"
plot "two-spiral.fann_rprop.test.out" title "fann rprop test" with lines -1, \
@@ -221,6 +239,7 @@ plot "two-spiral.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"two-spiral.fann_quickprop.test.out" title "fann quickprop test" with lines 3, \
"two-spiral.fann_batch.test.out" title "fann batch test" with lines 4, \
"two-spiral.fann_incremental.test.out" title "fann incremental test" with lines 5, \
+ "two-spiral.fann_incremental_momentum.test.out" title "fann incr mom test" with lines 6, \
"two-spiral.lwnn.test.out" title "lwnn incremental test" with lines 8, \
"two-spiral.jneural.test.out" title "jneural incremental test" with lines 9, \
"two-spiral.fann_rprop.train.out" title "fann rprop train" with lines -1, \
@@ -229,6 +248,7 @@ plot "two-spiral.fann_rprop.test.out" title "fann rprop test" with lines -1, \
"two-spiral.fann_quickprop.train.out" title "fann quickprop train" with lines 3, \
"two-spiral.fann_batch.train.out" title "fann batch train" with lines 4, \
"two-spiral.fann_incremental.train.out" title "fann incremental train" with lines 5, \
+ "two-spiral.fann_incremental_momentum.train.out" title "fann incr mom train" with lines 6, \
"two-spiral.lwnn.train.out" title "lwnn incremental train" with lines 8, \
"two-spiral.jneural.train.out" title "jneural incremental train" with lines 9
set nologscale
diff --git a/benchmarks/quality.cc b/benchmarks/quality.cc
index f24925b..5f89aa9 100644
--- a/benchmarks/quality.cc
+++ b/benchmarks/quality.cc
@@ -159,7 +159,8 @@ void quality_benchmark_fann(bool stepwise, fann_train_enum training_algorithm,
FILE * train_out, FILE * test_out,
unsigned int num_input, unsigned int num_neurons_hidden1,
unsigned int num_neurons_hidden2, unsigned int num_output,
- unsigned int seconds_of_training, double seconds_between_reports)
+ unsigned int seconds_of_training, double seconds_between_reports,
+ float momentum = 0)
{
float train_error, test_error;
unsigned int i, decimal_point, j, train_bit_fail, test_bit_fail;
@@ -181,6 +182,7 @@ void quality_benchmark_fann(bool stepwise, fann_train_enum training_algorithm,
}
fann_set_training_algorithm(ann, training_algorithm);
+ fann_set_learning_momentum(ann, momentum);
if(stepwise)
{
@@ -498,6 +500,14 @@ int main(int argc, char *argv[])
num_neurons_hidden2, train_data->num_output,
seconds_of_training, seconds_between_reports);
}
+ else if(strcmp(argv[1], "fann_incremental_momentum") == 0)
+ {
+ quality_benchmark_fann(false, FANN_TRAIN_INCREMENTAL, NULL, train_data, test_data,
+ train_out, test_out,
+ train_data->num_input, num_neurons_hidden1,
+ num_neurons_hidden2, train_data->num_output,
+ seconds_of_training, seconds_between_reports, 0.4);
+ }
else if(strcmp(argv[1], "fann_quickprop") == 0)
{
quality_benchmark_fann(false, FANN_TRAIN_QUICKPROP, NULL, train_data, test_data,
diff --git a/examples/robot.c b/examples/momentums.c
similarity index 58%
copy from examples/robot.c
copy to examples/momentums.c
index 35bacb3..6514930 100644
--- a/examples/robot.c
+++ b/examples/momentums.c
@@ -9,12 +9,12 @@ version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA02111-1307USA
*/
#include <stdio.h>
@@ -31,39 +31,31 @@ int main()
unsigned int i = 0;
- printf("Creating network.\n");
+ float momentum;
train_data = fann_read_train_from_file("../benchmarks/datasets/robot.train");
+ test_data = fann_read_train_from_file("../benchmarks/datasets/robot.test");
- ann = fann_create_standard(num_layers,
- train_data->num_input, num_neurons_hidden, train_data->num_output);
+ for ( momentum = 0.0; momentum < 0.7; momentum += 0.1 )
+ {
+ printf("============= momentum = %f =============\n", momentum);
- printf("Training network.\n");
+ ann = fann_create_standard(num_layers,
+ train_data->num_input, num_neurons_hidden, train_data->num_output);
- fann_set_training_algorithm(ann, FANN_TRAIN_INCREMENTAL);
- fann_set_training_algorithm(ann, FANN_TRAIN_QUICKPROP);
+ fann_set_training_algorithm(ann, FANN_TRAIN_INCREMENTAL);
- fann_train_on_data(ann, train_data, 3000, 10, desired_error);
+ fann_set_learning_momentum(ann, momentum);
- printf("Testing network.\n");
+ fann_train_on_data(ann, train_data, 2000, 500, desired_error);
- test_data = fann_read_train_from_file("../benchmarks/datasets/robot.test");
+ printf("MSE error on train data: %f\n", fann_test_data(ann, train_data));
+ printf("MSE error on test data : %f\n", fann_test_data(ann, test_data));
- fann_reset_MSE(ann);
- for(i = 0; i < test_data->num_data; i++)
- {
- fann_test(ann, test_data->input[i], test_data->output[i]);
+ fann_destroy(ann);
}
- printf("MSE error on test data: %f\n", fann_get_MSE(ann));
- printf("Saving network.\n");
-
- fann_save(ann, "robot_float.net");
-
- printf("Cleaning up.\n");
fann_destroy_train(train_data);
fann_destroy_train(test_data);
- fann_destroy(ann);
-
return 0;
}
diff --git a/examples/robot.c b/examples/robot.c
index 35bacb3..f584b45 100644
--- a/examples/robot.c
+++ b/examples/robot.c
@@ -41,7 +41,7 @@ int main()
printf("Training network.\n");
fann_set_training_algorithm(ann, FANN_TRAIN_INCREMENTAL);
- fann_set_training_algorithm(ann, FANN_TRAIN_QUICKPROP);
+ fann_set_learning_momentum(ann, 0.4);
fann_train_on_data(ann, train_data, 3000, 10, desired_error);
diff --git a/src/fann.c b/src/fann.c
index 0765aae..3fccdb6 100644
--- a/src/fann.c
+++ b/src/fann.c
@@ -766,6 +766,7 @@ FANN_EXTERNAL void FANN_API fann_destroy(struct fann *ann)
fann_safe_free(ann->train_slopes);
fann_safe_free(ann->prev_train_slopes);
fann_safe_free(ann->prev_steps);
+ fann_safe_free(ann->prev_weights_deltas);
fann_safe_free(ann->errstr);
fann_safe_free(ann->cascade_activation_functions);
fann_safe_free(ann->cascade_activation_steepnesses);
@@ -974,6 +975,7 @@ FANN_EXTERNAL void FANN_API fann_print_parameters(struct fann *ann)
printf("Bit fail limit :%4d\n", ann->bit_fail_limit);
#else
printf("Learning rate : %5.2f\n", ann->learning_rate);
+ printf("Learning momentum : %5.2f\n", ann->learning_momentum);
printf("Quickprop decay : %9.6f\n", ann->quickprop_decay);
printf("Quickprop mu : %5.2f\n", ann->quickprop_mu);
printf("RPROP increase factor : %5.2f\n", ann->rprop_increase_factor);
@@ -1100,6 +1102,7 @@ struct fann *fann_allocate_structure(unsigned int num_layers)
ann->error_log = NULL;
ann->errstr = NULL;
ann->learning_rate = 0.7;
+ ann->learning_momentum = 0.0;
ann->total_neurons = 0;
ann->total_connections = 0;
ann->num_input = 0;
@@ -1108,6 +1111,7 @@ struct fann *fann_allocate_structure(unsigned int num_layers)
ann->train_slopes = NULL;
ann->prev_steps = NULL;
ann->prev_train_slopes = NULL;
+ ann->prev_weights_deltas = NULL;
ann->training_algorithm = FANN_TRAIN_RPROP;
ann->num_MSE = 0;
ann->MSE_value = 0;
@@ -1172,7 +1176,7 @@ struct fann *fann_allocate_structure(unsigned int num_layers)
ann->rprop_delta_min = 0.0;
ann->rprop_delta_max = 50.0;
ann->rprop_delta_zero = 0.5;
-
+
fann_init_error_data((struct fann_error *) ann);
#ifdef FIXEDFANN
diff --git a/src/fann_train.c b/src/fann_train.c
index 005e62c..f68d070 100644
--- a/src/fann_train.c
+++ b/src/fann_train.c
@@ -354,22 +354,36 @@ void fann_backpropagate_MSE(struct fann *ann)
void fann_update_weights(struct fann *ann)
{
struct fann_neuron *neuron_it, *last_neuron, *prev_neurons, **connections;
- fann_type tmp_error, *weights;
+ fann_type tmp_error, delta_w, *weights;
struct fann_layer *layer_it;
unsigned int i;
unsigned int num_connections;
/* store some variabels local for fast access */
const float learning_rate = ann->learning_rate;
+ const float learning_momentum = ann->learning_momentum;
struct fann_neuron *first_neuron = ann->first_layer->first_neuron;
struct fann_layer *first_layer = ann->first_layer;
const struct fann_layer *last_layer = ann->last_layer;
fann_type *error_begin = ann->train_errors;
+ fann_type *deltas_begin, *weights_deltas;
+
+ /* if no room allocated for the deltas, allocate it now */
+ if(ann->prev_weights_deltas == NULL)
+ {
+ ann->prev_weights_deltas =
+ (fann_type *) calloc(ann->total_connections_allocated, sizeof(fann_type));
+ if(ann->prev_weights_deltas == NULL)
+ {
+ fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+ return;
+ }
+ }
#ifdef DEBUGTRAIN
printf("\nupdate weights\n");
#endif
-
+ deltas_begin = ann->prev_weights_deltas;
prev_neurons = first_neuron;
for(layer_it = (first_layer + 1); layer_it != last_layer; layer_it++)
{
@@ -388,9 +402,12 @@ void fann_update_weights(struct fann *ann)
tmp_error = error_begin[neuron_it - first_neuron] * learning_rate;
num_connections = neuron_it->last_con - neuron_it->first_con;
weights = ann->weights + neuron_it->first_con;
+ weights_deltas = deltas_begin + neuron_it->first_con;
for(i = 0; i != num_connections; i++)
{
- weights[i] += tmp_error * prev_neurons[i].value;
+ delta_w = tmp_error * prev_neurons[i].value + learning_momentum * weights_deltas[i];
+ weights[i] += delta_w ;
+ weights_deltas[i] = delta_w;
}
}
}
@@ -402,9 +419,12 @@ void fann_update_weights(struct fann *ann)
num_connections = neuron_it->last_con - neuron_it->first_con;
weights = ann->weights + neuron_it->first_con;
connections = ann->connections + neuron_it->first_con;
+ weights_deltas = deltas_begin + neuron_it->first_con;
for(i = 0; i != num_connections; i++)
{
- weights[i] += tmp_error * connections[i]->value;
+ delta_w = tmp_error * prev_neurons[i].value + learning_momentum * weights_deltas[i];
+ weights[i] += delta_w;
+ weights_deltas[i] = delta_w;
}
}
}
@@ -922,3 +942,4 @@ FANN_GET_SET(float, rprop_delta_min)
FANN_GET_SET(float, rprop_delta_max)
FANN_GET_SET(enum fann_stopfunc_enum, train_stop_function)
FANN_GET_SET(fann_type, bit_fail_limit)
+FANN_GET_SET(float, learning_momentum)
diff --git a/src/include/fann_data.h b/src/include/fann_data.h
index f7f108c..4744ce7 100644
--- a/src/include/fann_data.h
+++ b/src/include/fann_data.h
@@ -409,6 +409,9 @@ struct fann
/* the learning rate of the network */
float learning_rate;
+ /* The learning momentum used for backpropagation algorithm. */
+ float learning_momentum;
+
/* the connection rate of the network
* between 0 and 1, 1 meaning fully connected
*/
@@ -621,7 +624,8 @@ struct fann
/* The initial stepsize */
float rprop_delta_zero;
-
+
+
/* Used to contain the slope errors used during batch training
* Is allocated during first training session,
* which means that if we do not train, it is never allocated.
@@ -637,6 +641,13 @@ struct fann
* Not allocated if not used.
*/
fann_type *prev_train_slopes;
+
+ /* The last delta applied to a connection weight.
+ * This is used for the momentum term in the backpropagation algorithm.
+ * Not allocated if not used.
+ */
+ fann_type *prev_weights_deltas;
+
};
#endif
diff --git a/src/include/fann_train.h b/src/include/fann_train.h
index 1d0529e..34261d8 100644
--- a/src/include/fann_train.h
+++ b/src/include/fann_train.h
@@ -464,6 +464,36 @@ FANN_EXTERNAL float FANN_API fann_get_learning_rate(struct fann *ann);
*/
FANN_EXTERNAL void FANN_API fann_set_learning_rate(struct fann *ann, float learning_rate);
+/* Function: fann_get_learning_momentum
+
+ Get the learning momentum.
+
+ The learning momentum can be used to speed up FANN_TRAIN_INCREMENTAL training.
+ A too high momentum will however not benefit training. Setting momentum to 0 will
+ be the same as not using the momentum parameter. The recommended value of this parameter
+ is between 0.0 and 1.0.
+
+ The default momentum is 0.
+
+ See also:
+ <fann_set_learning_momentum>, <fann_set_training_algorithm>
+
+ This function appears in FANN >= 2.0.0.
+ */
+FANN_EXTERNAL float FANN_API fann_get_learning_momentum(struct fann *ann);
+
+
+/* Function: fann_set_learning_momentum
+
+ Set the learning momentum.
+
+ More info available in <fann_get_learning_momentum>
+
+ This function appears in FANN >= 2.0.0.
+ */
+FANN_EXTERNAL void FANN_API fann_set_learning_momentum(struct fann *ann, float learning_momentum);
+
+
/* Function: fann_set_activation_function
Set the activation function for neuron number *neuron* in layer number *layer*,
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/libfann.git
More information about the debian-science-commits
mailing list