AI-assisted Pipeline Diagnostics and Inspection w/ mmWave

1/////////////////////////////////////////////  
2        //    AI-assisted Pipeline Diagnostics     //
3       //      and Crack Inspection w/ mmWave     //
4      //             ---------------             //
5     //          (Arduino Nicla Vision)         //           
6    //             by Kutluhan Aktar           // 
7   //                                         //
8  /////////////////////////////////////////////
9
10//
11// Export data items from a 60GHz mmWave sensor, train a NN to diagnose pipeline issues, and inspect model results w/ output images on a web app.
12//
13// For more information:
14// https://www.theamplituhedron.com/projects/AI_assisted_Pipeline_Diagnostics_and_Crack_Inspection_w_mmWave/
15//
16//
17// Connections
18// Arduino Nicla Vision :  
19//                                Arduino Nano
20// UART_TX (PA_9)   --------------- A0
21// UART_RX (PA_10)  --------------- A1
22
23
24// Include the required libraries:
25#include <WiFi.h>
26#include "camera.h"
27#include "gc2145.h"
28
29// Include the Edge Impulse model converted to an Arduino library:
30#include <AI-assisted_Pipeline_Diagnostics_inferencing.h>
31
32// Define the required parameters to run an inference with the Edge Impulse model.
33#define FREQUENCY_HZ        EI_CLASSIFIER_FREQUENCY
34#define INTERVAL_MS         (1000 / (FREQUENCY_HZ + 1))
35
36// Define the features array to classify one frame of data.
37float features[EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE];
38size_t feature_ix = 0;
39
40// Define the threshold value for the model outputs (predictions).
41float threshold = 0.60;
42
43// Define the pipeline diagnostic class names:
44String classes[] = {"Clogged", "Cracked", "Leakage"};
45
46char ssid[] = "<_SSID_>";        // your network SSID (name)
47char pass[] = "<_PASSWORD_>";    // your network password (use for WPA, or use as key for WEP)
48int keyIndex = 0;                // your network key Index number (needed only for WEP)
49
50// Define the server on LattePanda 3 Delta.
51char server[] = "192.168.1.22";
52// Define the web application path.
53String application = "/pipeline_diagnostics_interface/update_server.php";
54
55// Initialize the WiFiClient object.
56WiFiClient client; /* WiFiSSLClient client; */
57
58// Define the required camera settings for the 2-megapixel CMOS camera (GC2145).
59GC2145 galaxyCore;
60Camera cam(galaxyCore);
61
62// Define the camera frame buffer.
63FrameBuffer fb;
64
65// Create a struct (data) including all 60GHz mmWave sensor data parameters:
66struct data {
67  float p1;
68  float p2;
69  float p3;
70  float p4;
71  float p5;
72  float p6;
73  float p7;
74};
75
76// Define the data holders:
77struct data mm;
78int predicted_class = -1;
79String data_packet = "";
80int del_1, del_2, del_3, del_4, del_5, del_6;
81
82void setup(){
83  Serial.begin(115200);
84
85  // Initialize the hardware serial port (Serial1) to communicate with Arduino Nano via serial communication. 
86  Serial1.begin(115200, SERIAL_8N1);
87
88  // Connect to WPA/WPA2 network. Change this line if using open or WEP network:
89  WiFi.begin(ssid, pass);
90  // Attempt to connect to the Wi-Fi network:
91  while(WiFi.status() != WL_CONNECTED){
92    // Wait for the connection:
93    delay(500);
94    Serial.print(".");
95  }
96  // If connected to the network successfully:
97  Serial.println("Connected to the Wi-Fi network successfully!");
98
99  // Define the pixel format and the FPS settings.
100  // Then, initialize the GC2145 camera.
101  if (!cam.begin(CAMERA_R320x320, CAMERA_RGB565, 30)) { // CAMERA_R320x240, CAMERA_R320x320
102    Serial.println("GC2145 camera: initialization failed!");
103  }else{
104    Serial.println("GC2145 camera: initialized successfully!");
105  }
106}
107
108void loop(){
109  // Obtain the data packet and commands transferred by Arduino Nano via serial communication.
110  if(Serial1.available() > 0){
111    data_packet = Serial1.readString();
112  }
113
114  if(data_packet != ""){
115    
116    Serial.print("\nReceived Data Packet: "); Serial.println(data_packet+"\n");
117    
118    // If Arduino Nano sends the Data command via serial communication:
119    if(data_packet.startsWith("Data")){
120      // Glean information as substrings from the transferred data packet by Arduino Nano.
121      del_1 = data_packet.indexOf("&");
122      del_2 = data_packet.indexOf("&", del_1 + 1);
123      String data_record = data_packet.substring(del_1 + 1, del_2);
124      String selected_class = data_packet.substring(del_2 + 1);
125
126      // Create the request string.
127      String request = "?data=OK&mmWave=" + String(data_record)
128                     + "&class=" + String(selected_class);
129      // Send the obtained mmWave data parameters and the selected pipeline diagnostic class to the web application via an HTTP GET request.
130      make_a_get_post_request(false, request);
131    }
132    
133    // If Arduino Nano sends the Run command via serial communication:
134    if(data_packet.startsWith("Run")){
135      // Glean information as substrings from the transferred data packet by Arduino Nano.
136      del_1 = data_packet.indexOf("&");
137      String data_record = data_packet.substring(del_1 + 1);
138      // Elicit data items from the generated substring.
139      del_1 = data_record.indexOf(",");
140      del_2 = data_record.indexOf(",", del_1 + 1);
141      del_3 = data_record.indexOf(",", del_2 + 1);
142      del_4 = data_record.indexOf(",", del_3 + 1);
143      del_5 = data_record.indexOf(",", del_4 + 1);
144      del_6 = data_record.indexOf(",", del_5 + 1);
145      // Convert and store the elicited data items.
146      mm.p1 = data_record.substring(0, del_1).toFloat();
147      mm.p2 = data_record.substring(del_1 + 1, del_2).toFloat();
148      mm.p3 = data_record.substring(del_2 + 1, del_3).toFloat();
149      mm.p4 = data_record.substring(del_3 + 1, del_4).toFloat();
150      mm.p5 = data_record.substring(del_4 + 1, del_5).toFloat();
151      mm.p6 = data_record.substring(del_5 + 1, del_6).toFloat();
152      mm.p7 = data_record.substring(del_6 + 1).toFloat();
153
154      // Run the Edge Impulse model to make predictions on the pipeline diagnostic classes.
155     run_inference_to_make_predictions(1);
156
157      // Capture a picture with the GC2145 camera.
158      take_picture();
159
160      // Create the request string.
161      String request = "?results=OK&mmWave=" + String(data_record)
162                     + "&class=" + classes[predicted_class];
163      // Send the obtained mmWave data parameters, the recently captured image, and the model detection result to the web application via an HTTP POST request.
164      make_a_get_post_request(true, request);
165    }
166
167    // Clear the received data packet.
168    data_packet = "";
169  }
170}
171
172void run_inference_to_make_predictions(int multiply){
173  // Scale (normalize) data items depending on the given model:
174  float scaled_p1 = mm.p1;
175  float scaled_p2 = mm.p2;
176  float scaled_p3 = mm.p3;
177  float scaled_p4 = mm.p4;
178  float scaled_p5 = mm.p5;
179  float scaled_p6 = mm.p6;
180  float scaled_p7 = mm.p7;
181
182  // Copy the scaled data items to the features buffer.
183  // If required, multiply the scaled data items while copying them to the features buffer.
184  for(int i=0; i<multiply; i++){  
185    features[feature_ix++] = scaled_p1;
186    features[feature_ix++] = scaled_p2;
187    features[feature_ix++] = scaled_p3;
188    features[feature_ix++] = scaled_p4;
189    features[feature_ix++] = scaled_p5;
190    features[feature_ix++] = scaled_p6;
191    features[feature_ix++] = scaled_p7;
192  }
193
194  // Display the progress of copying data to the features buffer.
195  Serial.print("Features Buffer Progress: "); Serial.print(feature_ix); Serial.print(" / "); Serial.println(EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE);
196  
197  // Run inference:
198  if(feature_ix == EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE){    
199    ei_impulse_result_t result;
200    // Create a signal object from the features buffer (frame).
201    signal_t signal;
202    numpy::signal_from_buffer(features, EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE, &signal);
203    // Run the classifier:
204    EI_IMPULSE_ERROR res = run_classifier(&signal, &result, false);
205    ei_printf("\nrun_classifier returned: %d\n", res);
206    if(res != 0) return;
207
208    // Print the inference timings on the serial monitor.
209    ei_printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n", 
210        result.timing.dsp, result.timing.classification, result.timing.anomaly);
211
212    // Obtain the prediction results for each label (class).
213    for(size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++){
214      // Print the prediction results on the serial monitor.
215      ei_printf("%s:\t%.5f\n", result.classification[ix].label, result.classification[ix].value);
216      // Get the predicted label (class).
217      if(result.classification[ix].value >= threshold) predicted_class = ix;
218    }
219    Serial.print("\nPredicted Class: "); Serial.println(predicted_class);
220
221    // Detect anomalies, if any:
222    #if EI_CLASSIFIER_HAS_ANOMALY == 1
223      ei_printf("Anomaly : \t%.3f\n", result.anomaly);
224    #endif
225
226    // Clear the features buffer (frame):
227    feature_ix = 0;
228  }
229}
230
231void make_a_get_post_request(bool post, String request){
232  // Connect to the web application named pipeline_diagnostics_interface. Change '80' with '443' if you are using SSL connection.
233  if (client.connect(server, 80)){
234    // If successful:
235    Serial.println("\nConnected to the web application successfully!\n");
236    // Create the query string:
237    String query = application + request;
238    // Transfer information to the web application via an HTTP POST or GET request depending on the given data parameter type.
239    if(post){
240      // Make an HTTP POST request:
241      String head = "--PipeDetection\r\nContent-Disposition: form-data; name=\"captured_image\"; filename=\"new_image.txt\"\r\nContent-Type: text/plain\r\n\r\n";
242      String tail = "\r\n--PipeDetection--\r\n";
243      // Get the total message length.
244      uint32_t totalLen = head.length() + cam.frameSize() + tail.length();
245      // Start the request:
246      client.println("POST " + query + " HTTP/1.1");
247      client.println("Host: 192.168.1.22");
248      client.println("Content-Length: " + String(totalLen));
249      client.println("Content-Type: multipart/form-data; boundary=PipeDetection");
250      client.println();
251      client.print(head);
252      client.write(fb.getBuffer(), cam.frameSize());
253      client.print(tail);
254      client.println("Connection: close");
255      client.println();
256      // Wait until transferring the image buffer.
257      delay(3000);
258      // If successful:
259      Serial.println("HTTP POST => Data transfer completed!\n");
260    }else{
261      // Make an HTTP GET request:
262      // Start the request:
263      client.println("GET " + query + " HTTP/1.1");
264      client.println("Host: 192.168.1.22");
265      client.println("Connection: close");
266      client.println();
267      //client.println("Connection: close");
268      delay(2000);
269      // If successful:
270      Serial.println("HTTP GET => Data transfer completed!\n");
271    }
272  }else{
273    Serial.println("\nConnection failed to the web application!\n");
274    delay(2000);
275  }
276}
277
278void take_picture(){
279  // Capture a picture with the GC2145 camera.
280  // If successful:
281  if(cam.grabFrame(fb, 3000) == 0){
282   Serial.println("\nGC2145 camera: image captured successfully!");
283  }else{
284    Serial.println("\nGC2145 camera: image capture failed!");
285  }
286  delay(2000);
287}

1/////////////////////////////////////////////  
2        //    AI-assisted Pipeline Diagnostics     //
3       //      and Crack Inspection w/ mmWave     //
4      //             ---------------             //
5     //          (Arduino Nicla Vision)         //           
6    //             by Kutluhan Aktar           // 
7   //                                         //
8  /////////////////////////////////////////////
9
10//
11// Export data items from a 60GHz mmWave sensor, train a NN to diagnose pipeline issues, and inspect model results w/ output images on a web app.
12//
13// For more information:
14// https://www.theamplituhedron.com/projects/AI_assisted_Pipeline_Diagnostics_and_Crack_Inspection_w_mmWave/
15//
16//
17// Connections
18// Arduino Nano : 
19//                                Arduino Nicla Vision
20// A0   --------------------------- UART_TX (PA_9) 
21// A1   --------------------------- UART_RX (PA_10)
22//                                Seeed Studio 60GHz mmWave Sensor
23// A2   --------------------------- TX
24// A3   --------------------------- RX
25//                                2.8'' 240x320 TFT LCD Touch Screen (ILI9341)
26// D10  --------------------------- CS 
27// D9   --------------------------- RESET 
28// D8   --------------------------- D/C
29// D11  --------------------------- SDI (MOSI)
30// D13  --------------------------- SCK 
31// 3.3V --------------------------- LED 
32// D12  --------------------------- SDO(MISO)
33//                                Control Button (A)
34// D2   --------------------------- +
35//                                Control Button (B)
36// D4   --------------------------- +
37//                                Control Button (C)
38// D7   --------------------------- +
39//                                Control Button (D)
40// A4   --------------------------- +
41//                                5mm Common Anode RGB LED
42// D3   --------------------------- R
43// D5   --------------------------- G
44// D6   --------------------------- B  
45
46
47// Include the required libraries:
48#include <SoftwareSerial.h>
49#include <Adafruit_GFX.h>
50#include <Adafruit_ILI9341.h>
51#include <60ghzbreathheart.h>
52
53// Define the serial port (Serial1) to communicate with Arduino Nicla Vision via serial communication.
54SoftwareSerial Nicla(A0, A1); // RX, TX
55
56// Define the serial port (Serial2) to communicate with the 60GHz mmWave sensor via serial communication.
57SoftwareSerial mmWave(A2, A3); // RX, TX
58
59// Define the 60GHz mmWave sensor object.
60BreathHeart_60GHz radar = BreathHeart_60GHz(&mmWave);
61
62// Define the required pins for the 240x320 TFT LCD Touch Screen (ILI9341):
63#define TFT_CS   10
64#define TFT_RST  9
65#define TFT_DC   8
66
67// Use hardware SPI (on Nano, SCK, MISO, MOSI) and the above for DC/CS/RST.
68Adafruit_ILI9341 tft = Adafruit_ILI9341(TFT_CS, TFT_DC, TFT_RST);
69
70// Define the mmWave radar color scheme.
71uint16_t b = ILI9341_BLACK; uint16_t g = ILI9341_GREEN; uint16_t y = ILI9341_YELLOW; uint16_t r = ILI9341_RED;
72uint16_t * circle_colors[] = {g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b};
73// Define the menu button color schemes and names.
74uint16_t button_colors[4][2] = {{ILI9341_DARKGREY, ILI9341_BLUE}, {ILI9341_DARKGREY, ILI9341_YELLOW}, {ILI9341_DARKGREY, ILI9341_RED}, {ILI9341_DARKGREY, ILI9341_CYAN}};
75String button_names[] = {"A", "B", "C", "D"};
76
77// Define the pipeline diagnostic class names:
78String classes[] = {"Leakage", "Cracked", "Clogged"};
79
80// Define the RGB LED pins:
81#define redPin     3
82#define greenPin   5
83#define bluePin    6
84
85// Define the control buttons.
86#define button_A   2
87#define button_B   4
88#define button_C   7
89#define button_D   A4
90
91// Define the data holders:
92#define TFT_ROTATION  2
93String data_packet = "";
94volatile boolean command = false;
95
96void setup(){
97  Serial.begin(115200);
98
99  // Initialize the software serial ports (Serial1 and Serial2).
100  Nicla.begin(115200);
101  mmWave.begin(115200);
102
103  pinMode(button_A, INPUT_PULLUP);
104  pinMode(button_B, INPUT_PULLUP);
105  pinMode(button_C, INPUT_PULLUP);
106  pinMode(button_D, INPUT_PULLUP);
107  pinMode(redPin, OUTPUT);
108  pinMode(greenPin, OUTPUT);
109  pinMode(bluePin, OUTPUT);
110
111  // Activate the real-time data transmission mode of the mmWave sensor.
112  // radar.reset_func(); delay(1000);
113  radar.ModeSelect_fuc(1);
114  
115  // Initialize the TFT LCD Touch Screen (ILI9341):
116  tft.begin();
117  tft.setRotation(TFT_ROTATION);
118  tft.fillScreen(ILI9341_NAVY);
119  tft.setTextColor(ILI9341_WHITE);  tft.setTextSize(2);
120  tft.setCursor(10, 10);
121  tft.println("Initializing...");
122  delay(5000);
123  adjustColor(255,0,255);
124  
125  // Show the mmWave radar and menu buttons.
126  int s[4] = {0,0,0,0}; menu_buttons(40,10,5,s,true);
127  screen_radar(10);
128  delay(1000);
129}
130
131void loop(){
132  // Collect the data parameters generated by the 60GHz mmWave sensor.
133  collect_mmWave_data(true);
134
135  // Send the collected mmWave data and the selected pipeline diagnostic class to Arduino Nicla Vision via serial communication.
136  if(!digitalRead(button_A)) { Nicla.print("Data&" + data_packet + "&Leakage"); Serial.println("\nData Sent! Selected Class: Leakage\n"); adjustColor(0,0,255); delay(2000); adjustColor(255,0,255); int s[4] = {1,0,0,0}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); }
137  if(!digitalRead(button_B)) { Nicla.print("Data&" + data_packet + "&Cracked"); Serial.println("\nData Sent! Selected Class: Cracked\n"); adjustColor(255,255,0); delay(2000); adjustColor(255,0,255); int s[4] = {0,1,0,0}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); }
138  if(!digitalRead(button_C)) { Nicla.print("Data&" + data_packet + "&Clogged"); Serial.println("\nData Sent! Selected Class: Clogged\n"); adjustColor(255,0,0); delay(2000); adjustColor(255,0,255); int s[4] = {0,0,1,0}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); }
139
140  // Send the collected mmWave data parameters to Arduino Nicla Vision via serial communication so as to run the Edge Impulse neural network model.
141  if(!digitalRead(button_D)) { Nicla.print("Run&" + data_packet); Serial.println("\nData Parameters Transferred Successfully!\n"); adjustColor(0,255,255); delay(2000); adjustColor(255,0,255); int s[4] = {0,0,0,1}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); }
142
143  // Undo the menu button selection and clear the latest command.
144  if(command){
145    int s[4] = {0,0,0,0}; menu_buttons(40,10,5,s,false);
146    command = false;
147  }
148}
149
150void collect_mmWave_data(bool p){
151  // Clear the data_packet string.
152  data_packet = "";
153  
154  // Initiate the breath and heartbeat information output.
155  radar.Breath_Heart();
156  // Add the evaluated breath and heartbeat parameters to the data_packet string.
157  if(radar.sensor_report != 0x00){
158    if(radar.heart_rate){ data_packet += String(radar.heart_rate, DEC); }else{ data_packet += "0"; }
159    if(radar.breath_rate){ data_packet += "," + String(radar.breath_rate, DEC); }else{ data_packet += ",0"; }
160  }else{
161    data_packet += "0,0";
162  }
163  delay(500);             
164
165  // Initiate the measuring information output.
166  radar.HumanExis_Func();
167  if(radar.sensor_report != 0x00){
168    if(radar.bodysign_val){ data_packet += "," + String(radar.bodysign_val, DEC); }else{ data_packet += ",0"; }
169    if(radar.distance){ data_packet += "," + String(radar.distance, DEC); }else{ data_packet += ",0"; }
170    if(radar.Dir_x){ data_packet += "," + String(radar.Dir_x, DEC); }else{ data_packet += ",0"; }
171    if(radar.Dir_y){ data_packet += "," + String(radar.Dir_y, DEC); }else{ data_packet += ",0"; }
172    if(radar.Dir_z){ data_packet += "," + String(radar.Dir_z, DEC); }else{ data_packet += ",0"; }
173  }else{
174    data_packet += ",0,0,0,0,0";
175  }
176  delay(500);
177
178  // Print the collected mmWave data parameters.
179  if(p) Serial.println("mmWave Data Parameters: " + data_packet);
180}
181
182void screen_radar(int radius){
183  int w = tft.width();
184  int h = tft.height();
185  int x = w/2; int y = w/2;
186  int limit = w / (2*radius);
187  // Draw the mmWave radar data visualization.
188  for(int i=limit; i>0; i--){
189    tft.fillCircle(x, y, i*radius, circle_colors[(limit+1)-i]);
190    delay(150);
191  }
192}
193
194void menu_buttons(int a, int e, int offset, int _select[4], bool _init){
195  int w = tft.width();
196  int h = tft.height();
197  int b = (w-(4*a)) / 5;
198  int x = b;
199  int y = h - a - e;
200  // If required, clear the screen.
201  if(_init) tft.fillScreen(ILI9341_BLACK);
202  // Draw the menu buttons indicating the control button status.
203  for(int i=0; i<4; i++){
204    tft.fillRect(x+(i*(a+b)), y, a, a, ILI9341_LIGHTGREY);
205    tft.fillRect((x+(i*(a+b))+offset), y+offset, a-(2*offset), a-(2*offset), button_colors[i][_select[i]]);
206    tft.setTextSize(3);
207    tft.setCursor((x+(i*(a+b))+offset+8), y+offset+5);
208    tft.println(button_names[i]);
209  }
210  // Print the activated feature.
211  tft.fillRect(0, y-26, w, 25, ILI9341_BLACK);
212  tft.setTextSize(2);
213  tft.setCursor(20, y-25);
214  if(_select[0]) tft.println("Selected: " + classes[0]);
215  if(_select[1]) tft.println("Selected: " + classes[1]);
216  if(_select[2]) tft.println("Selected: " + classes[2]);
217  if(_select[3]) tft.println("EI Model Running!");
218}
219
220void adjustColor(int r, int g, int b){
221  analogWrite(redPin, (255-r));
222  analogWrite(greenPin, (255-g));
223  analogWrite(bluePin, (255-b));
224}