Merge pull request #229 from deephealthproject/develop

Minor fixes

CMakeLists.txt

@@ -9,7 +9,7 @@ cmake_minimum_required(VERSION 3.9.2)
 option(BUILD_SUPERBUILD "Compile using the superbuild system" OFF)
 option(BUILD_PROTOBUF "Compile using Protobuf" ON)
 option(BUILD_OPENMP "Compile using OpenMP" ON)
-option(BUILD_HPC "Compile using agressive flags" ON)
+option(BUILD_HPC "Compile using aggressive flags" ON)
 option(BUILD_TESTS "Compile tests (HCP needs to be disabled)" ON)  # Disable HCP to pass tests (there are numerical errors)
 option(USE_LOCAL_GTEST "Use the local library to avoid problems derived from the 'One Definition Rule'" ON)
 option(BUILD_EXAMPLES "Compile examples" ON)
@@ -21,6 +21,16 @@ if(WIN32)
     option(BUILD_SHARED_LIBS "" OFF) # Prefer lib over dll in windows
 endif()
 
+###########################################################################
+############################### WARNINGS ##################################
+###########################################################################
+
+# Issues with HPC and logic functions
+if(BUILD_TESTS AND BUILD_HPC)
+    message(WARNING "[WARNING] Some logic functions are not compatible with the 'BUILD_HPC' flag.
+If you're unit testing, or using one of these logic functions: isfinite(), isinf(), isnan(), isposinf(), isneginf(); then we recommend you to disable the HPC flag: '-D BUILD_HPC=OFF' to obtain the expected results.")
+endif()
+
 ###########################################################################
 ################################# BUILD ###################################
 ###########################################################################

examples/nn/1_mnist/10_mnist_rnn_func.cpp

@@ -53,9 +53,9 @@ int main(int argc, char **argv) {
           rmsprop(0.001), // Optimizer
           {"softmax_cross_entropy"}, // Losses
           {"categorical_accuracy"}, // Metrics
-          CS_GPU({1}) // one GPU
+          //CS_GPU({1}) // one GPU
           //CS_GPU({1,1},100) // two GPU with weight sync every 100 batches
-          //CS_CPU()
+          CS_CPU()
 	  //CS_FPGA({1})
     );
 
@@ -89,6 +89,7 @@ int main(int argc, char **argv) {
         next_batch({x_train,y_train},{x_train_batch,y_train_batch});
 
         x_train_batch->reshape_({batch_size,28,28}); // time x dim
+        y_train_batch->reshape_({batch_size,1,10});
 
         zeroGrads(net);
         forward(net,{x_train_batch});

examples/nn/1_mnist/14_mnist_losses.cpp

@@ -10,6 +10,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <iostream>
+#include <iomanip>
 
 #include "eddl/apis/eddl.h"
 
@@ -112,32 +113,30 @@ int main(int argc, char **argv) {
 
       for(j=0;j<num_batches;j++)  {
 
-        cout<<"Batch "<<j<<" ";
+        cout<<"Batch "<<j;
         next_batch({x_train},{batch});
 
         zeroGrads(net);
 
         forward(net,{batch});
 
-        diceploss+=compute_loss(dicep)/batch_size;
-        cout<<"diceploss="<<diceploss/(j+1)<<" ";
-        fflush(stdout);
+        diceploss+=compute_loss(dicep)/(float)batch_size;
+        cout<<" ( dice_pixel_loss="<< std::setprecision(3) << std::fixed << diceploss/(float)(j+1);
 
-        diceiloss+=compute_loss(dicei)/batch_size;
-        cout<<"diceiloss="<<diceiloss/(j+1)<<" ";
-        fflush(stdout);
+        diceiloss+=compute_loss(dicei)/(float)batch_size;
+        cout<<"; dice_img_loss="<< std::setprecision(3) << std::fixed << diceiloss/(float)(j+1);
+
+        mseloss+=compute_loss(mse)/(float)batch_size;
+        cout<<"; mse_loss=" << std::setprecision(3) << std::fixed <<  mseloss/(float)(j+1);
 
-        mseloss+=compute_loss(mse)/batch_size;
-        cout<<"mseloss="<<mseloss/(j+1)<<"\r";
+        cout <<" )" <<"\r";
         fflush(stdout);
 
         optimize(dicep);
         //optimize({mse,dicep});
 
 
         update(net);
-
-
       }
 
       printf("\n");

examples/nn/1_mnist/8_mnist_rnn.cpp

@@ -52,9 +52,9 @@ int main(int argc, char **argv) {
           rmsprop(0.001), // Optimizer
           {"softmax_cross_entropy"}, // Losses
           {"categorical_accuracy"}, // Metrics
-          CS_GPU({1}) // one GPU
+          //CS_GPU({1}) // one GPU
           //CS_GPU({1,1},100) // two GPU with weight sync every 100 batches
-          //CS_CPU()
+          CS_CPU()
 	  //CS_FPGA({1})
     );
 
@@ -70,7 +70,9 @@ int main(int argc, char **argv) {
 
     // Reshape to fit recurrent batch x timestep x dim
     x_train->reshape_({60000,28,28});
+    y_train->reshape_({60000,1,10});
     x_test->reshape_({10000,28,28});
+    y_test->reshape_({10000,1,10});
 
     // Preprocessing
     x_train->div_(255.0f);

examples/nn/4_NLP/1_nlp_sentiment_rnn.cpp

@@ -58,9 +58,9 @@ int main(int argc, char **argv) {
           opt, // Optimizer
           {"binary_cross_entropy"}, // Losses
           {"binary_accuracy"}, // Metrics
-          //CS_GPU({1}) // one GPU
+          CS_GPU({1}) // one GPU
           //CS_GPU({1,1},100) // two GPU with weight sync every 100 batches
-          CS_CPU()
+//          CS_CPU()
     );
 
     // View model

examples/nn/4_NLP/2_nlp_sentiment_lstm.cpp

@@ -57,9 +57,9 @@ int main(int argc, char **argv) {
           opt, // Optimizer
           {"binary_cross_entropy"}, // Losses
           {"binary_accuracy"}, // Metrics
-          //CS_GPU({1}) // one GPU
+          CS_GPU({1}) // one GPU
           //CS_GPU({1,1},100) // two GPU with weight sync every 100 batches
-          CS_CPU()
+//          CS_CPU()
     );
 
     // View model

examples/nn/4_NLP/3_nlp_machine_translation.cpp

@@ -80,9 +80,9 @@ int main(int argc, char **argv) {
           opt, // Optimizer
           {"softmax_cross_entropy"}, // Losses
           {"accuracy"}, // Metrics
-          //CS_GPU({1}) // one GPU
+          CS_GPU({1}) // one GPU
           //CS_GPU({1,1},100) // two GPU with weight sync every 100 batches
-          CS_CPU()
+//          CS_CPU()
     );
 
 

examples/nn/4_NLP/4_nlp_video_to_labels.cpp

@@ -79,7 +79,13 @@ int main(int argc, char **argv) {
     layer out = ReLu(l);
     model deepVO = Model({in},{out});
 
-    build(deepVO, adam(), {"mse"}, {"mse"},  CS_CPU() );
+    build(deepVO,
+          adam(),
+          {"mse"},
+          {"mse"},
+          CS_GPU({1})
+//          CS_CPU()
+          );
     plot(deepVO,"model.pdf","TB");
     summary(deepVO);
 

examples/nn/4_NLP/5_nlp_text_generation.cpp

@@ -116,9 +116,9 @@ int main(int argc, char **argv) {
           opt, // Optimizer
           {"softmax_cross_entropy"}, // Losses
           {"accuracy"}, // Metrics
-          //CS_GPU({1}) // one GPU
+          CS_GPU({1}) // one GPU
           //CS_GPU({1,1},100) // two GPU with weight sync every 100 batches
-          CS_CPU()
+//          CS_CPU()
     );
 
     // View model

src/CMakeLists.txt

@@ -408,6 +408,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/
 
 # SHOW OPTIONS
 message(STATUS "===========================================" )
+message(STATUS "============== CMAKE SUMMARY ==============" )
 message(STATUS "===========================================" )
 message(STATUS "Project name: " ${CMAKE_PROJECT_NAME} )
 message(STATUS "Project version: " ${CMAKE_PROJECT_VERSION} )
@@ -418,7 +419,9 @@ message(STATUS "Build target: " ${BUILD_TARGET} )
 message(STATUS "Build type: " ${CMAKE_BUILD_TYPE} )
 message(STATUS "Build shared libs: " ${BUILD_SHARED_LIBS} )
 message(STATUS "Build coverage: " ${BUILD_COVERAGE} )
-message(STATUS "Superbuild: " ${BUILD_SUPERBUILD} )
+message(STATUS "Build sanitizers: " ${BUILD_SANITIZERS} )
+message(STATUS "Build HPC: " ${BUILD_HPC} )
+message(STATUS "Use superbuild: " ${BUILD_SUPERBUILD} )
 message(STATUS "-------------------------------------------" )
 message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER_ID} (${CMAKE_CXX_COMPILER}) | Version: ${CMAKE_CXX_COMPILER_VERSION}")
 message(STATUS "C++ flags: " ${CMAKE_CXX_FLAGS})

src/serialization/onnx/eddl_onnx_export.cpp

@@ -1347,21 +1347,36 @@ using namespace std;
 		w->set_data_type( onnx::TensorProto::FLOAT );
 		vector<int> w_dims {1, 4*layer->units, layer->input->shape[1]}; // w_dims shape[0] = 1 beacuse is only forward
         w->mutable_dims()->Add( w_dims.begin(), w_dims.end() ); // Set the shape of the weights
-        w->mutable_float_data()->Add( layer->Wix->ptr, layer->Wix->ptr + layer->Wix->size ); // i weights
-        w->mutable_float_data()->Add( layer->Wox->ptr, layer->Wox->ptr + layer->Wox->size ); // o weights
-        w->mutable_float_data()->Add( layer->Wfx->ptr, layer->Wfx->ptr + layer->Wfx->size ); // f weights
-        w->mutable_float_data()->Add( layer->Wcx->ptr, layer->Wcx->ptr + layer->Wcx->size ); // c weights
+		/*
+		 * The Weights are permuted before saving them (required by ONNX standad)
+		 */
+		Tensor* Wix = layer->Wix->permute({1, 0});
+        w->mutable_float_data()->Add( Wix->ptr, Wix->ptr + Wix->size ); // i weights
+		Tensor* Wox = layer->Wox->permute({1, 0});
+        w->mutable_float_data()->Add( Wox->ptr, Wox->ptr + Wox->size ); // o weights
+		Tensor* Wfx = layer->Wfx->permute({1, 0});
+        w->mutable_float_data()->Add( Wfx->ptr, Wfx->ptr + Wfx->size ); // f weights
+		Tensor* Wcx = layer->Wcx->permute({1, 0});
+        w->mutable_float_data()->Add( Wcx->ptr, Wcx->ptr + Wcx->size ); // c weights
 
 		// R input (recurrent weights for all the layers W[iofc])
 		onnx::TensorProto* r = graph->add_initializer();
 		r->set_name( layer->name + "_R" );
 		r->set_data_type( onnx::TensorProto::FLOAT );
 		vector<int> r_dims {1, 4*layer->units, layer->units}; // r_dims shape[0] = 1 beacuse is only forward
         r->mutable_dims()->Add( r_dims.begin(), r_dims.end() ); // Set the shape of the weights
-        r->mutable_float_data()->Add( layer->Wih->ptr, layer->Wih->ptr + layer->Wih->size ); // i recurrent weights
-        r->mutable_float_data()->Add( layer->Woh->ptr, layer->Woh->ptr + layer->Woh->size ); // o recurrent weights
-        r->mutable_float_data()->Add( layer->Wfh->ptr, layer->Wfh->ptr + layer->Wfh->size ); // f recurrent weights
-        r->mutable_float_data()->Add( layer->Wch->ptr, layer->Wch->ptr + layer->Wch->size ); // c recurrent weights
+		/*
+		 * The Weights are permuted before saving them (required by ONNX standad)
+		 */
+		Tensor* Wih = layer->Wih->permute({1, 0});
+        r->mutable_float_data()->Add( Wih->ptr, Wih->ptr + Wih->size ); // i recurrent weights
+		Tensor* Woh = layer->Woh->permute({1, 0});
+        r->mutable_float_data()->Add( Woh->ptr, Woh->ptr + Woh->size ); // o recurrent weights
+		Tensor* Wfh = layer->Wfh->permute({1, 0});
+        r->mutable_float_data()->Add( Wfh->ptr, Wfh->ptr + Wfh->size ); // f recurrent weights
+		Tensor* Wch = layer->Wch->permute({1, 0});
+        r->mutable_float_data()->Add( Wch->ptr, Wch->ptr + Wch->size ); // c recurrent weights
+
 		// B input (biases for all the layers)
 		onnx::TensorProto* b = graph->add_initializer();
 		b->set_name( layer->name + "_B" );

src/serialization/onnx/eddl_onnx_import.cpp

@@ -1605,7 +1605,9 @@ using namespace std;
 						vector<int> dims_g = map_init_dims[weights_gates];
                         int input_size = dims_g[2];
 
-						vector<int> dims_input_lstm = {dims_g[2], dims_g[1]/4};
+						// Load input weights with shape [hidden_size, input_size]. After load we transpose
+						//    Note: EDDL input weights are of shape [input_size, hidden_size]
+						vector<int> dims_input_lstm = {dims_g[1]/4, dims_g[2]};
 
 						vector<float>* weights_input_g = new vector<float>;
 						vector<float>* weights_output_g = new vector<float>;
@@ -1634,43 +1636,53 @@ using namespace std;
 						recurrence_weights_cell_g->assign(  recurrence_weights_g->begin() + w_size * 3  , recurrence_weights_g->begin() + w_size * 4);
 
 						LLSTM* lstm = new LLSTM({parent}, hidden_size, 0, 0, name, dev, mem);
-
+						/*
+						* The Weights are permuted before copying them to the LSTM layer (mismatch between ONNX standad and EDDL implementation)
+						*/
 						Tensor* weights_input_tensor = new Tensor(dims_input_lstm, NEW_FROM_VECTOR_PTR(weights_input_g), dev);
+						weights_input_tensor->permute_({1, 0});
 						Tensor::copy(weights_input_tensor, lstm->Wix );
 						delete weights_input_tensor;
 						delete weights_input_g;
 
 						Tensor* weights_output_tensor = new Tensor(dims_input_lstm, NEW_FROM_VECTOR_PTR(weights_output_g), dev);
+						weights_output_tensor->permute_({1, 0});
 						Tensor::copy(weights_output_tensor, lstm->Wox );
 						delete weights_output_tensor;
 						delete weights_output_g;
 
 						Tensor* weights_forget_tensor = new Tensor(dims_input_lstm, NEW_FROM_VECTOR_PTR(weights_forget_g), dev);
+						weights_forget_tensor->permute_({1, 0});
 						Tensor::copy(weights_forget_tensor, lstm->Wfx );
 						delete weights_forget_tensor;
 						delete weights_forget_g;
 
 						Tensor* weights_cell_tensor = new Tensor(dims_input_lstm, NEW_FROM_VECTOR_PTR(weights_cell_g), dev);
+						weights_cell_tensor->permute_({1, 0});
 						Tensor::copy(weights_cell_tensor, lstm->Wcx );
 						delete weights_cell_tensor;
 						delete weights_cell_g;
 
 						Tensor* recurrence_weights_input_tensor = new Tensor(dims_recurrent_lstm, NEW_FROM_VECTOR_PTR(recurrence_weights_input_g), dev);
+						recurrence_weights_input_tensor->permute_({1, 0});
 						Tensor::copy(recurrence_weights_input_tensor, lstm->Wih );
 						delete recurrence_weights_input_tensor;
 						delete recurrence_weights_input_g;
 
 						Tensor* recurrence_weights_output_tensor = new Tensor(dims_recurrent_lstm, NEW_FROM_VECTOR_PTR(recurrence_weights_output_g), dev);
+						recurrence_weights_output_tensor->permute_({1, 0});
 						Tensor::copy(recurrence_weights_output_tensor, lstm->Woh );
 						delete recurrence_weights_output_tensor;
 						delete recurrence_weights_output_g;
 
 						Tensor* recurrence_weights_forget_tensor = new Tensor(dims_recurrent_lstm, NEW_FROM_VECTOR_PTR(recurrence_weights_forget_g), dev);
+						recurrence_weights_forget_tensor->permute_({1, 0});
 						Tensor::copy(recurrence_weights_forget_tensor, lstm->Wfh );
 						delete recurrence_weights_forget_tensor;
 						delete recurrence_weights_forget_g;
 
 						Tensor* recurrence_weights_cell_tensor = new Tensor(dims_recurrent_lstm, NEW_FROM_VECTOR_PTR(recurrence_weights_cell_g), dev);
+						recurrence_weights_cell_tensor->permute_({1, 0});
 						Tensor::copy(recurrence_weights_cell_tensor, lstm->Wch );
 						delete recurrence_weights_cell_tensor;
 						delete recurrence_weights_cell_g;