Game reactive LEDs with Python, OpenCV and Arduino

Using OpenCV screen detection, trigger LEDs based on something on your screen

Dec 2, 2022

•

3443 views

•

0 respects

lights

games

Components and supplies

Male/Female Jumper Wires

RGB Diffused Common Cathode

Resistor 220 ohm

Jumper wires (generic)

Breadboard (generic)

Arduino UNO

Project description

Code

mainDetection

python

The object detection loops that utilize openCV and sends the commands to arduino

1import cv2 as cv
2import numpy as np
3import os
4import time
5import  serial
6import keyboard
7
8arduinoData = serial.Serial('com3',115200)
9
10#  imports screenshot function from other file
11from windowCapture import WindowCapture
12    
13wincap = WindowCapture(None)
14
15#Starts a timer to check for fps of  the program
16loop_time = time.time()
17
18while(True):
19    screenshot= wincap.get_screenshot()
20    
21    #can be adjusted to needs
22    threshold = 0.9
23    
24    #renamed  the max variable based on the ult name that is being detected
25    RCneedle_img  = cv.imread('Raincutter.png',cv.IMREAD_UNCHANGED)
26    result = cv.matchTemplate(screenshot,RCneedle_img,  cv.TM_CCOEFF_NORMED)
27    min_val, RCmax_val, min_loc, max_loc = cv.minMaxLoc(result)
28
29    DBneedle_img = cv.imread('DandelionBreeze.png',cv.IMREAD_UNCHANGED)
30    result  = cv.matchTemplate(screenshot,DBneedle_img, cv.TM_CCOEFF_NORMED)
31    min_val,  DBmax_val, min_loc, max_loc = cv.minMaxLoc(result)
32
33    SSneedle_img = cv.imread('SakuraSwirl.png',cv.IMREAD_UNCHANGED)
34    result = cv.matchTemplate(screenshot,SSneedle_img, cv.TM_CCOEFF_NORMED)
35    min_val, SSmax_val, min_loc, max_loc = cv.minMaxLoc(result)
36
37    ROTneedle_img  = cv.imread('ROTermination.png',cv.IMREAD_UNCHANGED)
38    result = cv.matchTemplate(screenshot,ROTneedle_img,  cv.TM_CCOEFF_NORMED)
39    min_val, ROTmax_val, min_loc, max_loc = cv.minMaxLoc(result)
40
41    if (RCmax_val>=threshold):
42        cmd = "RC" + '\r'
43
44    
45    elif(DBmax_val>=threshold):
46        cmd = "DB" + '\r'
47        
48    
49    elif(ROTmax_val>=threshold):
50        cmd = "ROT" + '\r'
51        
52
53    elif(SSmax_val>=threshold):
54        cmd = "SS" + '\r'
55         
56
57    else:
58        cmd = "No  ult" + '\r'
59        
60
61                
62
63
64
65    #after cmd  is assigned a value, sent to arduino
66    arduinoData.write(cmd.encode())
67    
68    #This prints the fps 
69    print(f"FPS {1 / (time.time() - loop_time)}  ")
70    
71    loop_time=time.time()
72    
73  
74    # prints the command  sent to the arduino for debugging purposes
75    print(cmd)
76    
77    #killswitch  for program, use any key
78    if keyboard.is_pressed('u'):
79        break
80
81
82
83print('Done')

import cv2 as cv
import numpy as np
import os
import time
import  serial
import keyboard

arduinoData = serial.Serial('com3',115200)

#  imports screenshot function from other file
from windowCapture import WindowCapture
    
wincap = WindowCapture(None)

#Starts a timer to check for fps of  the program
loop_time = time.time()

while(True):
    screenshot= wincap.get_screenshot()
    
    #can be adjusted to needs
    threshold = 0.9
    
    #renamed  the max variable based on the ult name that is being detected
    RCneedle_img  = cv.imread('Raincutter.png',cv.IMREAD_UNCHANGED)
    result = cv.matchTemplate(screenshot,RCneedle_img,  cv.TM_CCOEFF_NORMED)
    min_val, RCmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    DBneedle_img = cv.imread('DandelionBreeze.png',cv.IMREAD_UNCHANGED)
    result  = cv.matchTemplate(screenshot,DBneedle_img, cv.TM_CCOEFF_NORMED)
    min_val,  DBmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    SSneedle_img = cv.imread('SakuraSwirl.png',cv.IMREAD_UNCHANGED)
    result = cv.matchTemplate(screenshot,SSneedle_img, cv.TM_CCOEFF_NORMED)
    min_val, SSmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    ROTneedle_img  = cv.imread('ROTermination.png',cv.IMREAD_UNCHANGED)
    result = cv.matchTemplate(screenshot,ROTneedle_img,  cv.TM_CCOEFF_NORMED)
    min_val, ROTmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    if (RCmax_val>=threshold):
        cmd = "RC" + '\r'

    
    elif(DBmax_val>=threshold):
        cmd = "DB" + '\r'
        
    
    elif(ROTmax_val>=threshold):
        cmd = "ROT" + '\r'
        

    elif(SSmax_val>=threshold):
        cmd = "SS" + '\r'
         

    else:
        cmd = "No  ult" + '\r'
        

                



    #after cmd  is assigned a value, sent to arduino
    arduinoData.write(cmd.encode())
    
    #This prints the fps 
    print(f"FPS {1 / (time.time() - loop_time)}  ")
    
    loop_time=time.time()
    
  
    # prints the command  sent to the arduino for debugging purposes
    print(cmd)
    
    #killswitch  for program, use any key
    if keyboard.is_pressed('u'):
        break



print('Done')

Arduino File

arduino

The code that allows the Arduino to read the commands from the serial port and triggers the LEDs

1int rPin=6;
2int gPin=3;
3int bPin=5;
4
5int rPin2=11;
6int  gPin2=10;
7int bPin2=9;
8
9int rVal;
10int gVal;
11int bVal;
12
13int  rVal2;
14int gVal2;
15int bVal2;
16
17
18String mode;
19
20void setup() {
21  // put your setup code here, to run once:
22Serial.begin(115200);
23pinMode(rPin,OUTPUT);
24pinMode(gPin,OUTPUT);
25pinMode(bPin,OUTPUT);
26pinMode(rPin2,OUTPUT);
27pinMode(gPin2,OUTPUT);
28pinMode(bPin2,OUTPUT);
29}
30
31void  loop() {
32  // put your main code here, to run repeatedly:
33
34// reads the  command from the serial
35mode=Serial.readStringUntil('\r');
36
37
38if (mode=="RC"){
39analogWrite(rPin,0);
40analogWrite(gPin,0);
41analogWrite(bPin,70);
42analogWrite(rPin2,0);
43analogWrite(gPin2,0);
44analogWrite(bPin2,70);
45}
46
47else  if(mode== "SS"){
48analogWrite(rPin,10);
49analogWrite(gPin,20);
50analogWrite(bPin,100);
51analogWrite(rPin2,10);
52analogWrite(gPin2,20);
53analogWrite(bPin2,100);
54
55}
56else  if (mode=="ROT"){
57analogWrite(rPin,10);
58analogWrite(gPin,20);
59analogWrite(bPin,100);
60analogWrite(rPin2,10);
61analogWrite(gPin2,20);
62analogWrite(bPin2,100);
63}
64
65else  if (mode=="DB"){
66analogWrite(rPin,10);
67analogWrite(gPin,70);
68analogWrite(bPin,0);
69analogWrite(rPin2,10);
70analogWrite(gPin2,70);
71analogWrite(bPin2,0);
72}
73
74else{
75analogWrite(rPin,0);
76analogWrite(gPin,0);
77analogWrite(bPin,0);
78analogWrite(rPin2,0);
79analogWrite(gPin2,0);
80analogWrite(bPin2,0);
81}
82
83}
84

int rPin=6;
int gPin=3;
int bPin=5;

int rPin2=11;
int  gPin2=10;
int bPin2=9;

int rVal;
int gVal;
int bVal;

int  rVal2;
int gVal2;
int bVal2;


String mode;

void setup() {
  // put your setup code here, to run once:
Serial.begin(115200);
pinMode(rPin,OUTPUT);
pinMode(gPin,OUTPUT);
pinMode(bPin,OUTPUT);
pinMode(rPin2,OUTPUT);
pinMode(gPin2,OUTPUT);
pinMode(bPin2,OUTPUT);
}

void  loop() {
  // put your main code here, to run repeatedly:

// reads the  command from the serial
mode=Serial.readStringUntil('\r');


if (mode=="RC"){
analogWrite(rPin,0);
analogWrite(gPin,0);
analogWrite(bPin,70);
analogWrite(rPin2,0);
analogWrite(gPin2,0);
analogWrite(bPin2,70);
}

else  if(mode== "SS"){
analogWrite(rPin,10);
analogWrite(gPin,20);
analogWrite(bPin,100);
analogWrite(rPin2,10);
analogWrite(gPin2,20);
analogWrite(bPin2,100);

}
else  if (mode=="ROT"){
analogWrite(rPin,10);
analogWrite(gPin,20);
analogWrite(bPin,100);
analogWrite(rPin2,10);
analogWrite(gPin2,20);
analogWrite(bPin2,100);
}

else  if (mode=="DB"){
analogWrite(rPin,10);
analogWrite(gPin,70);
analogWrite(bPin,0);
analogWrite(rPin2,10);
analogWrite(gPin2,70);
analogWrite(bPin2,0);
}

else{
analogWrite(rPin,0);
analogWrite(gPin,0);
analogWrite(bPin,0);
analogWrite(rPin2,0);
analogWrite(gPin2,0);
analogWrite(bPin2,0);
}

}

templateMatching

python

Used to get an idea of the needle location and confidence values

1import cv2 as cv 
2
3#reads the images
4haystack_img = cv.imread('haystack.png',cv.IMREAD_UNCHANGED)
5needle_img  = cv.imread('needle.png',cv.IMREAD_UNCHANGED)
6
7#assigns the results to variables
8result  = cv.matchTemplate(haystack_img,needle_img, cv.TM_CCOEFF_NORMED)
9
10min_val,  max_val, min_loc, max_loc = cv.minMaxLoc(result)
11
12
13threshold = 0.9
14
15if  max_val >= threshold:
16    print('Found needle')
17    print(f'Best match top  left position: {max_loc}')
18    print(f'Best match confidence: {max_val}')
19
20    #gets dimensions of needle image 
21    needle_w = needle_img.shape[1]
22    needle_h = needle_img.shape[0]
23    
24    #dimensions of rectangle
25    top_left  = max_loc
26    bottom_right = (top_left[0] + needle_w, top_left[1] + needle_h  )
27    
28    #draws rectangle over where the image is detected
29    cv.rectangle(haystack_img,  top_left, bottom_right,
30    color=(0,255,0), thickness=2, lineType=cv.LINE_4)
31    
32    #displays result until a key is pressed
33    cv.imshow('Result', haystack_img)
34    cv.waitKey()
35
36  
37else:
38    print('Needle not found')
39

import cv2 as cv 

#reads the images
haystack_img = cv.imread('haystack.png',cv.IMREAD_UNCHANGED)
needle_img  = cv.imread('needle.png',cv.IMREAD_UNCHANGED)

#assigns the results to variables
result  = cv.matchTemplate(haystack_img,needle_img, cv.TM_CCOEFF_NORMED)

min_val,  max_val, min_loc, max_loc = cv.minMaxLoc(result)


threshold = 0.9

if  max_val >= threshold:
    print('Found needle')
    print(f'Best match top  left position: {max_loc}')
    print(f'Best match confidence: {max_val}')

    #gets dimensions of needle image 
    needle_w = needle_img.shape[1]
    needle_h = needle_img.shape[0]
    
    #dimensions of rectangle
    top_left  = max_loc
    bottom_right = (top_left[0] + needle_w, top_left[1] + needle_h  )
    
    #draws rectangle over where the image is detected
    cv.rectangle(haystack_img,  top_left, bottom_right,
    color=(0,255,0), thickness=2, lineType=cv.LINE_4)
    
    #displays result until a key is pressed
    cv.imshow('Result', haystack_img)
    cv.waitKey()

  
else:
    print('Needle not found')

windowCapture

python

Uses the win32api to take rapid screenshots and return them

1import numpy as np
2import os
3import time
4
5import win32gui,  win32ui, win32con
6
7class WindowCapture:
8    
9    #leave w,h as 0 for  init
10    #hwnd can be changed to your window name
11    w = 0
12    h = 0
13    hwnd = None
14
15    def __init__(self,window_name):
16
17        self.hwnd  = win32gui.FindWindow(None, window_name)
18        if window_name is None:
19            self.hwnd = win32gui.GetDesktopWindow()  
20        if not self.hwnd:
21            raise Exception(f'Window not found:{window_name}')
22        
23        #  Size of the section of the screen that is being captured
24        # Put monitor  resolution for whole screen 
25        self.h = 80
26        self.w = 80
27
28
29    def get_screenshot(self):
30        
31
32        
33        
34        wDC  = win32gui.GetWindowDC(self.hwnd)
35        dcObj=win32ui.CreateDCFromHandle(wDC)
36        cDC=dcObj.CreateCompatibleDC()
37        dataBitMap = win32ui.CreateBitmap()
38        dataBitMap.CreateCompatibleBitmap(dcObj, self.w, self.h)
39        cDC.SelectObject(dataBitMap)
40        #Coordinates of the upper left corner of the section captured
41        #  Put 0,0 for whole screen
42        cDC.BitBlt((0,0),(self.w, self.h) , dcObj, (1779,932),  win32con.SRCCOPY)
43    
44        #Screenshot is unrecogonizable but this block  of code converts it
45        signedIntsArray = dataBitMap.GetBitmapBits(True)
46        img = np.fromstring(signedIntsArray, dtype='uint8')
47        img.shape  = (self.h ,self.w ,4)
48
49
50        # Frees resources by deleting frame data  after each loop
51        dcObj.DeleteDC()
52        cDC.DeleteDC()
53        win32gui.ReleaseDC(self.hwnd,  wDC)
54        win32gui.DeleteObject(dataBitMap.GetHandle())
55
56        return  img

import numpy as np
import os
import time

import win32gui,  win32ui, win32con

class WindowCapture:
    
    #leave w,h as 0 for  init
    #hwnd can be changed to your window name
    w = 0
    h = 0
    hwnd = None

    def __init__(self,window_name):

        self.hwnd  = win32gui.FindWindow(None, window_name)
        if window_name is None:
            self.hwnd = win32gui.GetDesktopWindow()  
        if not self.hwnd:
            raise Exception(f'Window not found:{window_name}')
        
        #  Size of the section of the screen that is being captured
        # Put monitor  resolution for whole screen 
        self.h = 80
        self.w = 80


    def get_screenshot(self):
        

        
        
        wDC  = win32gui.GetWindowDC(self.hwnd)
        dcObj=win32ui.CreateDCFromHandle(wDC)
        cDC=dcObj.CreateCompatibleDC()
        dataBitMap = win32ui.CreateBitmap()
        dataBitMap.CreateCompatibleBitmap(dcObj, self.w, self.h)
        cDC.SelectObject(dataBitMap)
        #Coordinates of the upper left corner of the section captured
        #  Put 0,0 for whole screen
        cDC.BitBlt((0,0),(self.w, self.h) , dcObj, (1779,932),  win32con.SRCCOPY)
    
        #Screenshot is unrecogonizable but this block  of code converts it
        signedIntsArray = dataBitMap.GetBitmapBits(True)
        img = np.fromstring(signedIntsArray, dtype='uint8')
        img.shape  = (self.h ,self.w ,4)


        # Frees resources by deleting frame data  after each loop
        dcObj.DeleteDC()
        cDC.DeleteDC()
        win32gui.ReleaseDC(self.hwnd,  wDC)
        win32gui.DeleteObject(dataBitMap.GetHandle())

        return  img

Downloadable files

Wiring diagram

Simple RGB setup using the 6 available PWM pins

Wiring diagram

Comments

Only logged in users can leave comments

v1ckery

0 Followers

•

0 Projects

Intro

Trademarks & CopyrightsWhistleblowingDigital Services ActTerms of ServicePrivacy PolicySecurityCookie Settings

Report content