Components and supplies
Mini Pushbutton
Elecrow 8.8″ (1920*480) IPS Screen
5mm Common Anode RGB LED
Arduino Nano
PCBWay Custom PCB
LattePanda 3 Delta
Nicla Vision
Jumper Wires Standard
2.8″ 240x320 TFT LCD Touch Screen (ILI9341)
SparkFun Logic Level Converter - Bi-Directional
60GHz mmWave Sensor - Breathing and Heartbeat Module
Resistor 220 Ohm
Anycubic Kobra 2
Tools and machines
Soldering Iron
Hot Glue Gun
Apps and platforms
Thonny IDE
XAMPP Server
Arduino IDE
Fusion 360
KiCad
Ultimate Cura
Visual Studio 2017
Edge Impulse Studio
Project description
Code
AI_assisted_Pipeline_Diagnostics_run_model.ino
cpp
AI_assisted_Pipeline_Diagnostics_run_model.ino
1///////////////////////////////////////////// 2 // AI-assisted Pipeline Diagnostics // 3 // and Crack Inspection w/ mmWave // 4 // --------------- // 5 // (Arduino Nicla Vision) // 6 // by Kutluhan Aktar // 7 // // 8 ///////////////////////////////////////////// 9 10// 11// Export data items from a 60GHz mmWave sensor, train a NN to diagnose pipeline issues, and inspect model results w/ output images on a web app. 12// 13// For more information: 14// https://www.theamplituhedron.com/projects/AI_assisted_Pipeline_Diagnostics_and_Crack_Inspection_w_mmWave/ 15// 16// 17// Connections 18// Arduino Nicla Vision : 19// Arduino Nano 20// UART_TX (PA_9) --------------- A0 21// UART_RX (PA_10) --------------- A1 22 23 24// Include the required libraries: 25#include <WiFi.h> 26#include "camera.h" 27#include "gc2145.h" 28 29// Include the Edge Impulse model converted to an Arduino library: 30#include <AI-assisted_Pipeline_Diagnostics_inferencing.h> 31 32// Define the required parameters to run an inference with the Edge Impulse model. 33#define FREQUENCY_HZ EI_CLASSIFIER_FREQUENCY 34#define INTERVAL_MS (1000 / (FREQUENCY_HZ + 1)) 35 36// Define the features array to classify one frame of data. 37float features[EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE]; 38size_t feature_ix = 0; 39 40// Define the threshold value for the model outputs (predictions). 41float threshold = 0.60; 42 43// Define the pipeline diagnostic class names: 44String classes[] = {"Clogged", "Cracked", "Leakage"}; 45 46char ssid[] = "<_SSID_>"; // your network SSID (name) 47char pass[] = "<_PASSWORD_>"; // your network password (use for WPA, or use as key for WEP) 48int keyIndex = 0; // your network key Index number (needed only for WEP) 49 50// Define the server on LattePanda 3 Delta. 51char server[] = "192.168.1.22"; 52// Define the web application path. 53String application = "/pipeline_diagnostics_interface/update_server.php"; 54 55// Initialize the WiFiClient object. 56WiFiClient client; /* WiFiSSLClient client; */ 57 58// Define the required camera settings for the 2-megapixel CMOS camera (GC2145). 59GC2145 galaxyCore; 60Camera cam(galaxyCore); 61 62// Define the camera frame buffer. 63FrameBuffer fb; 64 65// Create a struct (data) including all 60GHz mmWave sensor data parameters: 66struct data { 67 float p1; 68 float p2; 69 float p3; 70 float p4; 71 float p5; 72 float p6; 73 float p7; 74}; 75 76// Define the data holders: 77struct data mm; 78int predicted_class = -1; 79String data_packet = ""; 80int del_1, del_2, del_3, del_4, del_5, del_6; 81 82void setup(){ 83 Serial.begin(115200); 84 85 // Initialize the hardware serial port (Serial1) to communicate with Arduino Nano via serial communication. 86 Serial1.begin(115200, SERIAL_8N1); 87 88 // Connect to WPA/WPA2 network. Change this line if using open or WEP network: 89 WiFi.begin(ssid, pass); 90 // Attempt to connect to the Wi-Fi network: 91 while(WiFi.status() != WL_CONNECTED){ 92 // Wait for the connection: 93 delay(500); 94 Serial.print("."); 95 } 96 // If connected to the network successfully: 97 Serial.println("Connected to the Wi-Fi network successfully!"); 98 99 // Define the pixel format and the FPS settings. 100 // Then, initialize the GC2145 camera. 101 if (!cam.begin(CAMERA_R320x320, CAMERA_RGB565, 30)) { // CAMERA_R320x240, CAMERA_R320x320 102 Serial.println("GC2145 camera: initialization failed!"); 103 }else{ 104 Serial.println("GC2145 camera: initialized successfully!"); 105 } 106} 107 108void loop(){ 109 // Obtain the data packet and commands transferred by Arduino Nano via serial communication. 110 if(Serial1.available() > 0){ 111 data_packet = Serial1.readString(); 112 } 113 114 if(data_packet != ""){ 115 116 Serial.print("\nReceived Data Packet: "); Serial.println(data_packet+"\n"); 117 118 // If Arduino Nano sends the Data command via serial communication: 119 if(data_packet.startsWith("Data")){ 120 // Glean information as substrings from the transferred data packet by Arduino Nano. 121 del_1 = data_packet.indexOf("&"); 122 del_2 = data_packet.indexOf("&", del_1 + 1); 123 String data_record = data_packet.substring(del_1 + 1, del_2); 124 String selected_class = data_packet.substring(del_2 + 1); 125 126 // Create the request string. 127 String request = "?data=OK&mmWave=" + String(data_record) 128 + "&class=" + String(selected_class); 129 // Send the obtained mmWave data parameters and the selected pipeline diagnostic class to the web application via an HTTP GET request. 130 make_a_get_post_request(false, request); 131 } 132 133 // If Arduino Nano sends the Run command via serial communication: 134 if(data_packet.startsWith("Run")){ 135 // Glean information as substrings from the transferred data packet by Arduino Nano. 136 del_1 = data_packet.indexOf("&"); 137 String data_record = data_packet.substring(del_1 + 1); 138 // Elicit data items from the generated substring. 139 del_1 = data_record.indexOf(","); 140 del_2 = data_record.indexOf(",", del_1 + 1); 141 del_3 = data_record.indexOf(",", del_2 + 1); 142 del_4 = data_record.indexOf(",", del_3 + 1); 143 del_5 = data_record.indexOf(",", del_4 + 1); 144 del_6 = data_record.indexOf(",", del_5 + 1); 145 // Convert and store the elicited data items. 146 mm.p1 = data_record.substring(0, del_1).toFloat(); 147 mm.p2 = data_record.substring(del_1 + 1, del_2).toFloat(); 148 mm.p3 = data_record.substring(del_2 + 1, del_3).toFloat(); 149 mm.p4 = data_record.substring(del_3 + 1, del_4).toFloat(); 150 mm.p5 = data_record.substring(del_4 + 1, del_5).toFloat(); 151 mm.p6 = data_record.substring(del_5 + 1, del_6).toFloat(); 152 mm.p7 = data_record.substring(del_6 + 1).toFloat(); 153 154 // Run the Edge Impulse model to make predictions on the pipeline diagnostic classes. 155 run_inference_to_make_predictions(1); 156 157 // Capture a picture with the GC2145 camera. 158 take_picture(); 159 160 // Create the request string. 161 String request = "?results=OK&mmWave=" + String(data_record) 162 + "&class=" + classes[predicted_class]; 163 // Send the obtained mmWave data parameters, the recently captured image, and the model detection result to the web application via an HTTP POST request. 164 make_a_get_post_request(true, request); 165 } 166 167 // Clear the received data packet. 168 data_packet = ""; 169 } 170} 171 172void run_inference_to_make_predictions(int multiply){ 173 // Scale (normalize) data items depending on the given model: 174 float scaled_p1 = mm.p1; 175 float scaled_p2 = mm.p2; 176 float scaled_p3 = mm.p3; 177 float scaled_p4 = mm.p4; 178 float scaled_p5 = mm.p5; 179 float scaled_p6 = mm.p6; 180 float scaled_p7 = mm.p7; 181 182 // Copy the scaled data items to the features buffer. 183 // If required, multiply the scaled data items while copying them to the features buffer. 184 for(int i=0; i<multiply; i++){ 185 features[feature_ix++] = scaled_p1; 186 features[feature_ix++] = scaled_p2; 187 features[feature_ix++] = scaled_p3; 188 features[feature_ix++] = scaled_p4; 189 features[feature_ix++] = scaled_p5; 190 features[feature_ix++] = scaled_p6; 191 features[feature_ix++] = scaled_p7; 192 } 193 194 // Display the progress of copying data to the features buffer. 195 Serial.print("Features Buffer Progress: "); Serial.print(feature_ix); Serial.print(" / "); Serial.println(EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE); 196 197 // Run inference: 198 if(feature_ix == EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE){ 199 ei_impulse_result_t result; 200 // Create a signal object from the features buffer (frame). 201 signal_t signal; 202 numpy::signal_from_buffer(features, EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE, &signal); 203 // Run the classifier: 204 EI_IMPULSE_ERROR res = run_classifier(&signal, &result, false); 205 ei_printf("\nrun_classifier returned: %d\n", res); 206 if(res != 0) return; 207 208 // Print the inference timings on the serial monitor. 209 ei_printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n", 210 result.timing.dsp, result.timing.classification, result.timing.anomaly); 211 212 // Obtain the prediction results for each label (class). 213 for(size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++){ 214 // Print the prediction results on the serial monitor. 215 ei_printf("%s:\t%.5f\n", result.classification[ix].label, result.classification[ix].value); 216 // Get the predicted label (class). 217 if(result.classification[ix].value >= threshold) predicted_class = ix; 218 } 219 Serial.print("\nPredicted Class: "); Serial.println(predicted_class); 220 221 // Detect anomalies, if any: 222 #if EI_CLASSIFIER_HAS_ANOMALY == 1 223 ei_printf("Anomaly : \t%.3f\n", result.anomaly); 224 #endif 225 226 // Clear the features buffer (frame): 227 feature_ix = 0; 228 } 229} 230 231void make_a_get_post_request(bool post, String request){ 232 // Connect to the web application named pipeline_diagnostics_interface. Change '80' with '443' if you are using SSL connection. 233 if (client.connect(server, 80)){ 234 // If successful: 235 Serial.println("\nConnected to the web application successfully!\n"); 236 // Create the query string: 237 String query = application + request; 238 // Transfer information to the web application via an HTTP POST or GET request depending on the given data parameter type. 239 if(post){ 240 // Make an HTTP POST request: 241 String head = "--PipeDetection\r\nContent-Disposition: form-data; name=\"captured_image\"; filename=\"new_image.txt\"\r\nContent-Type: text/plain\r\n\r\n"; 242 String tail = "\r\n--PipeDetection--\r\n"; 243 // Get the total message length. 244 uint32_t totalLen = head.length() + cam.frameSize() + tail.length(); 245 // Start the request: 246 client.println("POST " + query + " HTTP/1.1"); 247 client.println("Host: 192.168.1.22"); 248 client.println("Content-Length: " + String(totalLen)); 249 client.println("Content-Type: multipart/form-data; boundary=PipeDetection"); 250 client.println(); 251 client.print(head); 252 client.write(fb.getBuffer(), cam.frameSize()); 253 client.print(tail); 254 client.println("Connection: close"); 255 client.println(); 256 // Wait until transferring the image buffer. 257 delay(3000); 258 // If successful: 259 Serial.println("HTTP POST => Data transfer completed!\n"); 260 }else{ 261 // Make an HTTP GET request: 262 // Start the request: 263 client.println("GET " + query + " HTTP/1.1"); 264 client.println("Host: 192.168.1.22"); 265 client.println("Connection: close"); 266 client.println(); 267 //client.println("Connection: close"); 268 delay(2000); 269 // If successful: 270 Serial.println("HTTP GET => Data transfer completed!\n"); 271 } 272 }else{ 273 Serial.println("\nConnection failed to the web application!\n"); 274 delay(2000); 275 } 276} 277 278void take_picture(){ 279 // Capture a picture with the GC2145 camera. 280 // If successful: 281 if(cam.grabFrame(fb, 3000) == 0){ 282 Serial.println("\nGC2145 camera: image captured successfully!"); 283 }else{ 284 Serial.println("\nGC2145 camera: image capture failed!"); 285 } 286 delay(2000); 287}
AI_assisted_Pipeline_Diagnostics_data_collect.ino
cpp
AI_assisted_Pipeline_Diagnostics_data_collect.ino
1///////////////////////////////////////////// 2 // AI-assisted Pipeline Diagnostics // 3 // and Crack Inspection w/ mmWave // 4 // --------------- // 5 // (Arduino Nicla Vision) // 6 // by Kutluhan Aktar // 7 // // 8 ///////////////////////////////////////////// 9 10// 11// Export data items from a 60GHz mmWave sensor, train a NN to diagnose pipeline issues, and inspect model results w/ output images on a web app. 12// 13// For more information: 14// https://www.theamplituhedron.com/projects/AI_assisted_Pipeline_Diagnostics_and_Crack_Inspection_w_mmWave/ 15// 16// 17// Connections 18// Arduino Nano : 19// Arduino Nicla Vision 20// A0 --------------------------- UART_TX (PA_9) 21// A1 --------------------------- UART_RX (PA_10) 22// Seeed Studio 60GHz mmWave Sensor 23// A2 --------------------------- TX 24// A3 --------------------------- RX 25// 2.8'' 240x320 TFT LCD Touch Screen (ILI9341) 26// D10 --------------------------- CS 27// D9 --------------------------- RESET 28// D8 --------------------------- D/C 29// D11 --------------------------- SDI (MOSI) 30// D13 --------------------------- SCK 31// 3.3V --------------------------- LED 32// D12 --------------------------- SDO(MISO) 33// Control Button (A) 34// D2 --------------------------- + 35// Control Button (B) 36// D4 --------------------------- + 37// Control Button (C) 38// D7 --------------------------- + 39// Control Button (D) 40// A4 --------------------------- + 41// 5mm Common Anode RGB LED 42// D3 --------------------------- R 43// D5 --------------------------- G 44// D6 --------------------------- B 45 46 47// Include the required libraries: 48#include <SoftwareSerial.h> 49#include <Adafruit_GFX.h> 50#include <Adafruit_ILI9341.h> 51#include <60ghzbreathheart.h> 52 53// Define the serial port (Serial1) to communicate with Arduino Nicla Vision via serial communication. 54SoftwareSerial Nicla(A0, A1); // RX, TX 55 56// Define the serial port (Serial2) to communicate with the 60GHz mmWave sensor via serial communication. 57SoftwareSerial mmWave(A2, A3); // RX, TX 58 59// Define the 60GHz mmWave sensor object. 60BreathHeart_60GHz radar = BreathHeart_60GHz(&mmWave); 61 62// Define the required pins for the 240x320 TFT LCD Touch Screen (ILI9341): 63#define TFT_CS 10 64#define TFT_RST 9 65#define TFT_DC 8 66 67// Use hardware SPI (on Nano, SCK, MISO, MOSI) and the above for DC/CS/RST. 68Adafruit_ILI9341 tft = Adafruit_ILI9341(TFT_CS, TFT_DC, TFT_RST); 69 70// Define the mmWave radar color scheme. 71uint16_t b = ILI9341_BLACK; uint16_t g = ILI9341_GREEN; uint16_t y = ILI9341_YELLOW; uint16_t r = ILI9341_RED; 72uint16_t * circle_colors[] = {g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b,g,b,r,b,y,b}; 73// Define the menu button color schemes and names. 74uint16_t button_colors[4][2] = {{ILI9341_DARKGREY, ILI9341_BLUE}, {ILI9341_DARKGREY, ILI9341_YELLOW}, {ILI9341_DARKGREY, ILI9341_RED}, {ILI9341_DARKGREY, ILI9341_CYAN}}; 75String button_names[] = {"A", "B", "C", "D"}; 76 77// Define the pipeline diagnostic class names: 78String classes[] = {"Leakage", "Cracked", "Clogged"}; 79 80// Define the RGB LED pins: 81#define redPin 3 82#define greenPin 5 83#define bluePin 6 84 85// Define the control buttons. 86#define button_A 2 87#define button_B 4 88#define button_C 7 89#define button_D A4 90 91// Define the data holders: 92#define TFT_ROTATION 2 93String data_packet = ""; 94volatile boolean command = false; 95 96void setup(){ 97 Serial.begin(115200); 98 99 // Initialize the software serial ports (Serial1 and Serial2). 100 Nicla.begin(115200); 101 mmWave.begin(115200); 102 103 pinMode(button_A, INPUT_PULLUP); 104 pinMode(button_B, INPUT_PULLUP); 105 pinMode(button_C, INPUT_PULLUP); 106 pinMode(button_D, INPUT_PULLUP); 107 pinMode(redPin, OUTPUT); 108 pinMode(greenPin, OUTPUT); 109 pinMode(bluePin, OUTPUT); 110 111 // Activate the real-time data transmission mode of the mmWave sensor. 112 // radar.reset_func(); delay(1000); 113 radar.ModeSelect_fuc(1); 114 115 // Initialize the TFT LCD Touch Screen (ILI9341): 116 tft.begin(); 117 tft.setRotation(TFT_ROTATION); 118 tft.fillScreen(ILI9341_NAVY); 119 tft.setTextColor(ILI9341_WHITE); tft.setTextSize(2); 120 tft.setCursor(10, 10); 121 tft.println("Initializing..."); 122 delay(5000); 123 adjustColor(255,0,255); 124 125 // Show the mmWave radar and menu buttons. 126 int s[4] = {0,0,0,0}; menu_buttons(40,10,5,s,true); 127 screen_radar(10); 128 delay(1000); 129} 130 131void loop(){ 132 // Collect the data parameters generated by the 60GHz mmWave sensor. 133 collect_mmWave_data(true); 134 135 // Send the collected mmWave data and the selected pipeline diagnostic class to Arduino Nicla Vision via serial communication. 136 if(!digitalRead(button_A)) { Nicla.print("Data&" + data_packet + "&Leakage"); Serial.println("\nData Sent! Selected Class: Leakage\n"); adjustColor(0,0,255); delay(2000); adjustColor(255,0,255); int s[4] = {1,0,0,0}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); } 137 if(!digitalRead(button_B)) { Nicla.print("Data&" + data_packet + "&Cracked"); Serial.println("\nData Sent! Selected Class: Cracked\n"); adjustColor(255,255,0); delay(2000); adjustColor(255,0,255); int s[4] = {0,1,0,0}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); } 138 if(!digitalRead(button_C)) { Nicla.print("Data&" + data_packet + "&Clogged"); Serial.println("\nData Sent! Selected Class: Clogged\n"); adjustColor(255,0,0); delay(2000); adjustColor(255,0,255); int s[4] = {0,0,1,0}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); } 139 140 // Send the collected mmWave data parameters to Arduino Nicla Vision via serial communication so as to run the Edge Impulse neural network model. 141 if(!digitalRead(button_D)) { Nicla.print("Run&" + data_packet); Serial.println("\nData Parameters Transferred Successfully!\n"); adjustColor(0,255,255); delay(2000); adjustColor(255,0,255); int s[4] = {0,0,0,1}; menu_buttons(40,10,5,s,false); screen_radar(5); command = true; delay(2000); } 142 143 // Undo the menu button selection and clear the latest command. 144 if(command){ 145 int s[4] = {0,0,0,0}; menu_buttons(40,10,5,s,false); 146 command = false; 147 } 148} 149 150void collect_mmWave_data(bool p){ 151 // Clear the data_packet string. 152 data_packet = ""; 153 154 // Initiate the breath and heartbeat information output. 155 radar.Breath_Heart(); 156 // Add the evaluated breath and heartbeat parameters to the data_packet string. 157 if(radar.sensor_report != 0x00){ 158 if(radar.heart_rate){ data_packet += String(radar.heart_rate, DEC); }else{ data_packet += "0"; } 159 if(radar.breath_rate){ data_packet += "," + String(radar.breath_rate, DEC); }else{ data_packet += ",0"; } 160 }else{ 161 data_packet += "0,0"; 162 } 163 delay(500); 164 165 // Initiate the measuring information output. 166 radar.HumanExis_Func(); 167 if(radar.sensor_report != 0x00){ 168 if(radar.bodysign_val){ data_packet += "," + String(radar.bodysign_val, DEC); }else{ data_packet += ",0"; } 169 if(radar.distance){ data_packet += "," + String(radar.distance, DEC); }else{ data_packet += ",0"; } 170 if(radar.Dir_x){ data_packet += "," + String(radar.Dir_x, DEC); }else{ data_packet += ",0"; } 171 if(radar.Dir_y){ data_packet += "," + String(radar.Dir_y, DEC); }else{ data_packet += ",0"; } 172 if(radar.Dir_z){ data_packet += "," + String(radar.Dir_z, DEC); }else{ data_packet += ",0"; } 173 }else{ 174 data_packet += ",0,0,0,0,0"; 175 } 176 delay(500); 177 178 // Print the collected mmWave data parameters. 179 if(p) Serial.println("mmWave Data Parameters: " + data_packet); 180} 181 182void screen_radar(int radius){ 183 int w = tft.width(); 184 int h = tft.height(); 185 int x = w/2; int y = w/2; 186 int limit = w / (2*radius); 187 // Draw the mmWave radar data visualization. 188 for(int i=limit; i>0; i--){ 189 tft.fillCircle(x, y, i*radius, circle_colors[(limit+1)-i]); 190 delay(150); 191 } 192} 193 194void menu_buttons(int a, int e, int offset, int _select[4], bool _init){ 195 int w = tft.width(); 196 int h = tft.height(); 197 int b = (w-(4*a)) / 5; 198 int x = b; 199 int y = h - a - e; 200 // If required, clear the screen. 201 if(_init) tft.fillScreen(ILI9341_BLACK); 202 // Draw the menu buttons indicating the control button status. 203 for(int i=0; i<4; i++){ 204 tft.fillRect(x+(i*(a+b)), y, a, a, ILI9341_LIGHTGREY); 205 tft.fillRect((x+(i*(a+b))+offset), y+offset, a-(2*offset), a-(2*offset), button_colors[i][_select[i]]); 206 tft.setTextSize(3); 207 tft.setCursor((x+(i*(a+b))+offset+8), y+offset+5); 208 tft.println(button_names[i]); 209 } 210 // Print the activated feature. 211 tft.fillRect(0, y-26, w, 25, ILI9341_BLACK); 212 tft.setTextSize(2); 213 tft.setCursor(20, y-25); 214 if(_select[0]) tft.println("Selected: " + classes[0]); 215 if(_select[1]) tft.println("Selected: " + classes[1]); 216 if(_select[2]) tft.println("Selected: " + classes[2]); 217 if(_select[3]) tft.println("EI Model Running!"); 218} 219 220void adjustColor(int r, int g, int b){ 221 analogWrite(redPin, (255-r)); 222 analogWrite(greenPin, (255-g)); 223 analogWrite(bluePin, (255-b)); 224}
Downloadable files
Custom parts and enclosures
Custom parts and enclosures
https://www.hackster.io/kutluhan-aktar/ai-assisted-pipeline-diagnostics-and-inspection-w-mmwave-d048c3#cad
Documentation
Schematics
Schematics
https://www.hackster.io/kutluhan-aktar/ai-assisted-pipeline-diagnostics-and-inspection-w-mmwave-d048c3#schematics
Comments
Only logged in users can leave comments