AIfES - Inference Tutorial

1
2  /*
3  www.aifes.ai
4  https://github.com/Fraunhofer-IMS/AIfES_for_Arduino
5  Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and  Systems. All rights reserved.
6
7  AIfES is free software: you can redistribute  it and/or modify
8  it under the terms of the GNU General Public License as published  by
9  the Free Software Foundation, either version 3 of the License, or
10  (at  your option) any later version.
11  
12  This program is distributed in the hope  that it will be useful,
13  but WITHOUT ANY WARRANTY; without even the implied  warranty of
14  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  GNU General Public License for more details.
16  
17  You should have received  a copy of the GNU General Public License
18  along with this program.  If not,  see <https://www.gnu.org/licenses/>.
19
20  AIfES_Inference_tutorial
21  --------------------
22
23  Versions:
24    1.0.0   Initial version
25  
26  The sketch shows an example  of how the inference of an already trained network is performed. 
27  In the concrete  example, a neural network was trained to map an XOR gate. 
28  The neural network  was trained in Keras and the configuration including the weights was imported into  AIfES. 
29  The network structure is 2-3(Sigmoid)-1(Sigmoid) and Sigmoid is used  as activation function.
30  The calculation is done in float 32.
31  
32  XOR  truth table
33  Input    Output
34  0   0    0
35  0   1    1
36  1   0    1
37  1   1    0
38  
39  Tested on:
40    Arduino UNO
41    Arduino Nano 33 BLE  Sense
42   
43*/
44#include <aifes.h>
45
46#define INPUTS  2
47#define NEURONS  3
48#define OUTPUTS 1
49
50#define DATA_COUNT 4
51
52void setup() {
53  Serial.begin(115200);  //115200 baud rate (If necessary, change in the serial monitor)
54  while (!Serial);
55
56  Serial.println(F("AIfES inference tutorial"));
57  Serial.println(F("Type  >inference< to start"));
58
59}
60
61void loop() {
62
63  while(Serial.available()  > 0 ){
64    String str = Serial.readString();
65    if(str.indexOf("inference")  > -1){        //Keyword "inference"
66      
67      Serial.println(F("Result  LayeredWeights method"));
68      Inference_LayeredWeights();
69      
70      Serial.println(F(""));
71      Serial.println(F("Result FlatWeights method"));
72      Inference_FlatWeights();
73
74      Serial.println(F(""));
75      Serial.println(F("Result FlatWeights bytes  method"));
76      Inference_FlatWeights_bytes();
77    }
78    else{
79      Serial.println(F("unknown"));
80      Serial.println(F("Type >inference<"));
81    }
82  }
83
84}
85
86void  Inference_LayeredWeights(){
87
88    //Tensor for the input data
89    //Input  data for the XOR ANN
90    //float input_data[] = {0.0f, 0.0f, 0.0f, 1.0f, 1.0f,  0.0f, 1.0f, 1.0f};  // 1D array example
91    float input_data[DATA_COUNT][INPUTS]  = {
92        {0.0f, 0.0f},
93        {0.0f, 1.0f},
94        {1.0f, 0.0f},
95        {1.0f, 1.0f}
96    };
97
98    //Tensor for the input data
99    uint16_t  input_shape[] = {DATA_COUNT, INPUTS};        // Definition of the input shape
100    aitensor_t input_tensor;                              // Creation of the input  AIfES tensor
101    input_tensor.dtype = aif32;                           // Definition  of the used data type, here float with 32 bits, different ones are available
102    input_tensor.dim = 2;                                 // Dimensions of the tensor,  here 2 dimensions, as specified at input_shape
103    input_tensor.shape = input_shape;                     // Set the shape of the input_tensor
104    input_tensor.data  = input_data;                       // Assign the input_data array to the tensor.  It expects a pointer to the array where the data is stored
105
106    //Tensor for  the output data
107    float output_data[DATA_COUNT * OUTPUTS];              //  Output data
108    uint16_t output_shape[] = {DATA_COUNT, OUTPUTS};      // Definition  of the output shape
109    aitensor_t output_tensor_1;                           //  Creation of the output AIfES tensor
110    output_tensor_1.dtype = aif32;                        //  Definition of the used data type, here float with 32 bits, different ones are available
111    output_tensor_1.dim = 2;                              // Dimensions of the tensor,  here 2 dimensions, as specified at input_shape
112    output_tensor_1.shape = output_shape;                 // Set the shape of the output_tensor
113    output_tensor_1.data  = output_data;                   // Assign the output_data array to the tensor.  It expects a pointer to the array where the data is stored
114
115    // ----------------------------------  Layer definition ---------------------------------------
116
117    // Input layer
118    uint16_t input_layer_shape[] = {1, INPUTS};         // Definition of the input  layer shape (The 1 must remain here regardless of the number of data sets)
119    ailayer_input_t  input_layer;                        // Creation of the AIfES input layer
120    input_layer.input_dim  = 2;                          // Definition of the input dimension (Must fit to  the input tensor)
121    input_layer.input_shape = input_layer_shape;        //  Handover of the input layer shape
122
123    // Dense layer (hidden layer)
124    const  float weights_hidden_layer[] PROGMEM = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f,  -9.0155f};  //Hidden layer weights
125    const float bias_weights_hidden_layer[]  PROGMEM = {-2.9653f,  2.3677f, -1.5968f};                         //Hidden layer  bias weights
126    
127    ailayer_dense_t hidden_layer;                                                                             //  Creation of the AIfES hidden dense layer
128    hidden_layer.neurons = NEURONS;                                                                           // Number  of neurons
129    hidden_layer.weights.data = (void*)weights_hidden_layer;                                                  //  Passing the hidden layer weights. The (void*) is only needed if you use PROGMEM
130    hidden_layer.bias.data = (void*)bias_weights_hidden_layer;                                                //  Passing the hidden layer bias weights. The (void*) is only needed if you use PROGMEM
131
132
133    // Sigmoid activation function
134    ailayer_sigmoid_f32_t sigmoid_layer_1;
135
136    /* // Alternative activation functions
137    ailayer_relu_f32_t          relu_layer;
138    ailayer_sigmoid_f32_t       sigmoid_layer;
139    ailayer_tanh_f32_t          tanh_layer;
140    ailayer_softsign_f32_t      softsign_layer;
141    ailayer_leaky_relu_f32_t    leaky_relu_layer;
142    ailayer_elu_f32_t           elu_layer;
143    //Alpha  values
144    leaky_relu_layer.alpha = 0.01f;
145    elu_layer.alpha = 1.0f;
146    //Softmax
147    ailayer_softmax_f32_t       softmax_layer; // Only for the  output layer
148    */
149
150    // Output dense layer
151    const float weights_output_layer[]  PROGMEM = {12.0305f, -6.5858f, 11.9371f};  //Output dense layer weights
152    const  float bias_weights_output_layer[] PROGMEM = {-5.4247f};                 //Output  dense layer bias weights
153    
154    ailayer_dense_t output_layer;                                                 //  Creation of the AIfES ouput dense layer
155    output_layer.neurons = OUTPUTS;                                               //  Number of neurons
156    output_layer.weights.data = (void*) weights_output_layer;                     // Passing the output layer weights. The (void*) is only needed  if you use PROGMEM
157    output_layer.bias.data = (void*) bias_weights_output_layer;                   // Passing the output layer bias weights. The (void*) is only  needed if you use PROGMEM
158
159    // Sigmoid activation function
160    ailayer_sigmoid_f32_t  sigmoid_layer_2;
161
162    // --------------------------- Define the structure  of the model ----------------------------
163
164    aimodel_t model;  // AIfES  model
165    ailayer_t *x;     // Layer object from AIfES, contains the layers
166
167    // Passing the layers to the AIfES model
168    model.input_layer = ailayer_input_f32_default(&input_layer);
169    x = ailayer_dense_f32_default(&hidden_layer, model.input_layer);
170    x =  ailayer_sigmoid_f32_default(&sigmoid_layer_1, x);
171    x = ailayer_dense_f32_default(&output_layer,  x);
172    model.output_layer = ailayer_sigmoid_f32_default(&sigmoid_layer_2, x);
173
174    aialgo_compile_model(&model); // Compile the AIfES model
175
176    // --------------------------------  Allocate and schedule the working memory for inference ---------
177    uint32_t  memory_size = aialgo_sizeof_inference_memory(&model);
178    void *memory_ptr =  malloc(memory_size);
179    // Here is an alternative if no "malloc" should be  used
180    //byte memory_ptr[memory_size];
181
182    // Schedule the memory over  the model
183    aialgo_schedule_inference_memory(&model, memory_ptr, memory_size);
184
185     // ------------------------------------- Run the inference ------------------------------------
186    aialgo_inference_model(&model, &input_tensor, &output_tensor_1);
187
188    //  ------------------------------------- Print result ------------------------------------
189    Serial.print("Output 0: ");
190    Serial.println(output_data[0],6);
191    Serial.print("Output  1: ");
192    Serial.println(output_data[1],6);
193    Serial.print("Output 2:  ");
194    Serial.println(output_data[2],6);
195    Serial.print("Output 3: ");
196    Serial.println(output_data[3],6);
197
198
199    free(memory_ptr);
200
201}
202
203void  Inference_FlatWeights(){
204
205    //FlatWeights array
206    const float FlatWeights[]  PROGMEM = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f, -9.0155f, -2.9653f,  2.3677f,  -1.5968f, 12.0305f, -6.5858f, 11.9371f,-5.4247f};
207
208    //Tensor for the input  data
209    //float input_data[] = {0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 0.0f, 1.0f, 1.0f};    // Input data for the XOR ANN
210    float input_data[DATA_COUNT][INPUTS] =  {
211        {0.0f, 0.0f},
212        {0.0f, 1.0f},
213        {1.0f, 0.0f},
214        {1.0f, 1.0f}
215    };
216
217    //Tensor for the input data
218    uint16_t  input_shape[] = {DATA_COUNT, INPUTS};      // Definition of the input shape
219    aitensor_t input_tensor;                            // Creation of the input  AIfES tensor
220    input_tensor.dtype = aif32;                         // Definition  of the used data type, here float with 32 bits, different ones are available
221    input_tensor.dim = 2;                               // Dimensions of the tensor,  here 2 dimensions, as specified at input_shape
222    input_tensor.shape = input_shape;                   // Set the shape of the input_tensor
223    input_tensor.data  = input_data;                     // Assign the input_data array to the tensor.  It expects a pointer to the array where the data is stored
224
225    //Tensor for  the output data
226    float output_data[DATA_COUNT * OUTPUTS];            // Output  data
227    uint16_t output_shape[] = {DATA_COUNT, OUTPUTS};    // Definition of  the output shape
228    aitensor_t output_tensor_1;                         // Creation  of the output AIfES tensor
229    output_tensor_1.dtype = aif32;                      //  Definition of the used data type, here float with 32 bits, different ones are available
230    output_tensor_1.dim = 2;                            // Dimensions of the tensor,  here 2 dimensions, as specified at input_shape
231    output_tensor_1.shape = output_shape;               // Set the shape of the output_tensor
232    output_tensor_1.data  = output_data;                 // Assign the output_data array to the tensor. It  expects a pointer to the array where the data is stored
233
234    // ----------------------------------  Layer definition ---------------------------------------
235
236    // Input layer
237    uint16_t input_layer_shape[] = {1, INPUTS};         // Definition of the input  layer shape (The 1 must remain here regardless of the number of data sets)
238    ailayer_input_t  input_layer;                        // Creation of the AIfES input layer
239    input_layer.input_dim  = 2;                          // Definition of the input dimension (Must fit to  the input tensor)
240    input_layer.input_shape = input_layer_shape;        //  Handover of the input layer shape
241
242    // Dense layer (hidden layer)
243    ailayer_dense_t  hidden_layer;                       // Creation of the AIfES hidden dense layer
244    hidden_layer.neurons = NEURONS;                     // Number of neurons
245
246    // Sigmoid activation function
247    ailayer_sigmoid_f32_t sigmoid_layer_1;
248
249    /* // Alternative activation functions
250    ailayer_relu_f32_t          relu_layer;
251    ailayer_sigmoid_f32_t       sigmoid_layer;
252    ailayer_tanh_f32_t          tanh_layer;
253    ailayer_softsign_f32_t      softsign_layer;
254    ailayer_leaky_relu_f32_t    leaky_relu_layer;
255    ailayer_elu_f32_t           elu_layer;
256    //Alpha  values
257    leaky_relu_layer.alpha = 0.01f;
258    elu_layer.alpha = 1.0f;
259    //Softmax
260    ailayer_softmax_f32_t       softmax_layer; // Only for the  output layer
261    */
262
263    // Output dense layer
264    ailayer_dense_t output_layer;                       // Creation of the AIfES ouput dense layer
265    output_layer.neurons  = OUTPUTS;                     // Number of neurons
266
267    // Sigmoid activation  function
268    ailayer_sigmoid_f32_t sigmoid_layer_2;
269
270    // ---------------------------  Define the structure of the model ----------------------------
271
272    aimodel_t  model;                                    // AIfES model
273    ailayer_t *x;                                       //  Layer object from AIfES, contains the layers
274
275    // Passing the layers to  the AIfES model
276    model.input_layer = ailayer_input_f32_default(&input_layer);
277    x = ailayer_dense_f32_default(&hidden_layer, model.input_layer);
278    x =  ailayer_sigmoid_f32_default(&sigmoid_layer_1, x);
279    x = ailayer_dense_f32_default(&output_layer,  x);
280    model.output_layer = ailayer_sigmoid_f32_default(&sigmoid_layer_2, x);
281
282    aialgo_compile_model(&model); // Compile the AIfES model
283
284    // --------------------------------  Allocate and schedule the working memory for inference ---------
285
286    // Parameter  memory size in Byte
287    uint32_t parameter_memory_size = aialgo_sizeof_parameter_memory(&model);
288    
289    // Calculate the number of float weights
290    uint32_t FlatWeight_array_length  = parameter_memory_size / sizeof(float);
291    Serial.print(F("Length FlatWeight  array: "));
292    Serial.println(FlatWeight_array_length);
293    
294    //Check  the array length
295    if(FlatWeight_array_length != sizeof(FlatWeights)/sizeof(float))
296    {
297      Serial.println(F("ERROR!: Number of weights wrong"));
298    }
299    
300    aialgo_distribute_parameter_memory(&model, (void*) FlatWeights, parameter_memory_size);  //The (void*) is only needed if you use PROGMEM
301
302    uint32_t memory_size  = aialgo_sizeof_inference_memory(&model);
303    void *memory_ptr = malloc(memory_size);
304    // Here is an alternative if no "malloc" should be used
305    //byte memory_ptr[memory_size];
306
307    // Schedule the memory over the model
308    aialgo_schedule_inference_memory(&model,  memory_ptr, memory_size);
309
310     // ------------------------------------- Run  the inference ------------------------------------
311    aialgo_inference_model(&model,  &input_tensor, &output_tensor_1);
312
313    // -------------------------------------  Print result ------------------------------------
314    Serial.print("Output 0:  ");
315    Serial.println(output_data[0],6);
316    Serial.print("Output 1: ");
317    Serial.println(output_data[1],6);
318    Serial.print("Output 2: ");
319    Serial.println(output_data[2],6);
320    Serial.print("Output 3: ");
321    Serial.println(output_data[3],6);
322
323    free(memory_ptr);
324
325}
326
327void Inference_FlatWeights_bytes(){
328
329    //FlatWeights array
330    float FlatWeights[] = {-10.1164f, -8.4212f, 5.4396f,  7.297f, -7.6482f, -9.0155f, -2.9653f,  2.3677f, -1.5968f, 12.0305f, -6.5858f, 11.9371f,-5.4247f};
331
332    //Tensor for the input data
333    //float input_data[] = {0.0f, 0.0f, 0.0f,  1.0f, 1.0f, 0.0f, 1.0f, 1.0f};    // Input data for the XOR ANN
334    float input_data[DATA_COUNT][INPUTS]  = {
335        {0.0f, 0.0f},
336        {0.0f, 1.0f},
337        {1.0f, 0.0f},
338        {1.0f, 1.0f}
339    };
340
341    //Tensor for the input data
342    uint16_t  input_shape[] = {DATA_COUNT, INPUTS};      // Definition of the input shape
343    aitensor_t input_tensor;                            // Creation of the input  AIfES tensor
344    input_tensor.dtype = aif32;                         // Definition  of the used data type, here float with 32 bits, different ones are available
345    input_tensor.dim = 2;                               // Dimensions of the tensor,  here 2 dimensions, as specified at input_shape
346    input_tensor.shape = input_shape;                   // Set the shape of the input_tensor
347    input_tensor.data  = input_data;                     // Assign the input_data array to the tensor.  It expects a pointer to the array where the data is stored
348
349    //Tensor for  the output data
350    float output_data[DATA_COUNT * OUTPUTS];            // Output  data
351    uint16_t output_shape[] = {DATA_COUNT, OUTPUTS};    // Definition of  the output shape
352    aitensor_t output_tensor_1;                         // Creation  of the output AIfES tensor
353    output_tensor_1.dtype = aif32;                      //  Definition of the used data type, here float with 32 bits, different ones are available
354    output_tensor_1.dim = 2;                            // Dimensions of the tensor,  here 2 dimensions, as specified at input_shape
355    output_tensor_1.shape = output_shape;               // Set the shape of the output_tensor
356    output_tensor_1.data  = output_data;                 // Assign the output_data array to the tensor. It  expects a pointer to the array where the data is stored
357
358    // ----------------------------------  Layer definition ---------------------------------------
359
360    // Input layer
361    uint16_t input_layer_shape[] = {1, INPUTS};         // Definition of the input  layer shape (The 1 must remain here regardless of the number of data sets)
362    ailayer_input_t  input_layer;                        // Creation of the AIfES input layer
363    input_layer.input_dim  = 2;                          // Definition of the input dimension (Must fit to  the input tensor)
364    input_layer.input_shape = input_layer_shape;        //  Handover of the input layer shape
365
366    // Dense layer (hidden layer)
367    ailayer_dense_t  hidden_layer;                       // Creation of the AIfES hidden dense layer
368    hidden_layer.neurons = NEURONS;                     // Number of neurons
369
370    // Sigmoid activation function
371    ailayer_sigmoid_f32_t sigmoid_layer_1;
372
373    /* // Alternative activation functions
374    ailayer_relu_f32_t          relu_layer;
375    ailayer_sigmoid_f32_t       sigmoid_layer;
376    ailayer_tanh_f32_t          tanh_layer;
377    ailayer_softsign_f32_t      softsign_layer;
378    ailayer_leaky_relu_f32_t    leaky_relu_layer;
379    ailayer_elu_f32_t           elu_layer;
380    //Alpha  values
381    leaky_relu_layer.alpha = 0.01f;
382    elu_layer.alpha = 1.0f;
383    //Softmax
384    ailayer_softmax_f32_t       softmax_layer; // Only for the  output layer
385    */
386
387    // Output dense layer
388    ailayer_dense_t output_layer;                       // Creation of the AIfES ouput dense layer
389    output_layer.neurons  = OUTPUTS;                     // Number of neurons
390
391    // Sigmoid activation  function
392    ailayer_sigmoid_f32_t sigmoid_layer_2;
393
394    // ---------------------------  Define the structure of the model ----------------------------
395
396    aimodel_t  model;                                    // AIfES model
397    ailayer_t *x;                                       //  Layer object from AIfES, contains the layers
398
399    // Passing the layers to  the AIfES model
400    model.input_layer = ailayer_input_f32_default(&input_layer);
401    x = ailayer_dense_f32_default(&hidden_layer, model.input_layer);
402    x =  ailayer_sigmoid_f32_default(&sigmoid_layer_1, x);
403    x = ailayer_dense_f32_default(&output_layer,  x);
404    model.output_layer = ailayer_sigmoid_f32_default(&sigmoid_layer_2, x);
405
406    aialgo_compile_model(&model); // Compile the AIfES model
407
408    // --------------------------------  Allocate and schedule the working memory for inference ---------
409
410    // Parameter  memory size in Byte
411    uint32_t parameter_memory_size = aialgo_sizeof_parameter_memory(&model);
412    
413    // Calculate the number of float weights
414    uint32_t FlatWeight_array_length  = parameter_memory_size / sizeof(float);
415    Serial.print(F("Length FlatWeight  array: "));
416    Serial.println(FlatWeight_array_length);
417    
418    //Check  the array length
419    if(FlatWeight_array_length != sizeof(FlatWeights)/sizeof(float))
420    {
421      Serial.println(F("ERROR!: Number of weights wrong"));
422    }
423
424    // Array for the weights in uint8
425    uint8_t FlatWeights_byte[parameter_memory_size];
426
427    // Typecast uint_8 pointer
428    uint8_t *FlatWeights_byte_ptr = (uint8_t *)  FlatWeights;
429
430    uint32_t i = 0;
431
432    // Copy the values into the uint8  array
433    for (i = 0; i < parameter_memory_size; i++) {
434      FlatWeights_byte[i]  = FlatWeights_byte_ptr[i];
435    }
436  
437    aialgo_distribute_parameter_memory(&model,  (void*) FlatWeights_byte, parameter_memory_size); //The (void*) is only needed if  you use PROGMEM
438
439    uint32_t memory_size = aialgo_sizeof_inference_memory(&model);
440    void *memory_ptr = malloc(memory_size);
441    // Here is an alternative if  no "malloc" should be used
442    //byte memory_ptr[memory_size];
443
444    //  Schedule the memory over the model
445    aialgo_schedule_inference_memory(&model,  memory_ptr, memory_size);
446
447     // ------------------------------------- Run  the inference ------------------------------------
448    aialgo_inference_model(&model,  &input_tensor, &output_tensor_1);
449
450    // -------------------------------------  Print result ------------------------------------
451    Serial.print("Output 0:  ");
452    Serial.println(output_data[0],6);
453    Serial.print("Output 1: ");
454    Serial.println(output_data[1],6);
455    Serial.print("Output 2: ");
456    Serial.println(output_data[2],6);
457    Serial.print("Output 3: ");
458    Serial.println(output_data[3],6);
459
460    free(memory_ptr);
461
462}
463