Using Mediapipe to control Gripper

1import cv2
2import serial
3import mediapipe as mp
4import math
5import  numpy as np
6import time
7import pyfirmata
8
9#--------------
10
11board  = pyfirmata.Arduino('COM3')
12servo = board.get_pin('d:9:s')
13  
14
15mp_drawing  = mp.solutions.drawing_utils
16mp_drawing_styles = mp.solutions.drawing_styles
17mp_hands  = mp.solutions.hands
18
19
20
21# Webcam Setup
22wCam, hCam = 640, 480
23cam  = cv2.VideoCapture(0)
24cam.set(3,wCam)
25cam.set(4,hCam)
26
27
28with mp_hands.Hands(
29    model_complexity=0,
30    min_detection_confidence=0.5,
31    min_tracking_confidence=0.5)  as hands:
32
33  while cam.isOpened():
34    success, image = cam.read()
35
36    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
37    results = hands.process(image)
38    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
39    if results.multi_hand_landmarks:
40      for hand_landmarks in results.multi_hand_landmarks:
41        mp_drawing.draw_landmarks(
42            image,
43            hand_landmarks,
44            mp_hands.HAND_CONNECTIONS,
45            mp_drawing_styles.get_default_hand_landmarks_style(),
46            mp_drawing_styles.get_default_hand_connections_style()
47            )
48
49    # multi_hand_landmarks method for Finding postion of Hand  landmarks      
50    lmList = []
51    if results.multi_hand_landmarks:
52      myHand  = results.multi_hand_landmarks[0]
53      for id, lm in enumerate(myHand.landmark):
54        h, w, c = image.shape
55        cx, cy = int(lm.x * w), int(lm.y * h)
56        lmList.append([id, cx, cy])          
57
58    # Assigning variables for  Thumb and Index finger position
59    if len(lmList) != 0:
60      x1, y1 = lmList[4][1],  lmList[4][2]
61      x2, y2 = lmList[8][1], lmList[8][2]
62
63      # Marking  Thumb and Index finger
64      cv2.circle(image, (x1,y1),15,(255,255,255))  
65      cv2.circle(image, (x2,y2),15,(255,255,255))   
66      cv2.line(image,(x1,y1),(x2,y2),(255,0,0),3)
67      length = math.hypot(x2-x1,y2-y1)
68      if length < 50:
69        cv2.line(image,(x1,y1),(x2,y2),(0,0,0),3)
70
71      
72      Pos = np.interp(length, [50, 220], [0, 100])
73      Posgripper=  (round(Pos))
74      #print(Posgripper)
75      converted_Posgripper = str(Posgripper)
76      cv2.putText(image, str(Posgripper), (50, 60), cv2.FONT_HERSHEY_COMPLEX, 2,  (255, 0, 0))
77      #cv2.line(image, 320, 320, (0,0,0), 2)
78      Servopos=(100-Posgripper)
79      print (Servopos)
80      servo.write(Servopos) 
81
82    
83    cv2.imshow('handDetector',  image) 
84    if cv2.waitKey(1) & 0xFF == ord('q'):
85      break
86cam.release()

1#include <Servo.h>
2#include <Firmata.h>
3
4Servo servos[MAX_SERVOS];
5byte  servoPinMap[TOTAL_PINS];
6byte servoCount = 0;
7
8void analogWriteCallback(byte  pin, int value)
9{
10  if (IS_PIN_DIGITAL(pin)) {
11    servos[servoPinMap[pin]].write(value);
12  }
13}
14
15void systemResetCallback()
16{
17  servoCount = 0;
18}
19
20void  setup()
21{
22  byte pin;
23
24  Firmata.setFirmwareVersion(FIRMATA_FIRMWARE_MAJOR_VERSION,  FIRMATA_FIRMWARE_MINOR_VERSION);
25  Firmata.attach(ANALOG_MESSAGE, analogWriteCallback);
26  Firmata.attach(SYSTEM_RESET, systemResetCallback);
27
28  Firmata.begin(57600);
29  systemResetCallback();
30
31  // attach servos from first digital pin up to  max number of
32  // servos supported for the board
33  for (pin = 0; pin < TOTAL_PINS;  pin++) {
34    if (IS_PIN_DIGITAL(pin)) {
35      if (servoCount < MAX_SERVOS)  {
36        servoPinMap[pin] = servoCount;
37        servos[servoPinMap[pin]].attach(PIN_TO_DIGITAL(pin));
38        servoCount++;
39      }
40    }
41  }
42}
43
44void loop()
45{
46  while (Firmata.available())
47    Firmata.processInput();
48}

1#include <Servo.h>
2#include <Firmata.h>
3
4Servo servos[MAX_SERVOS];
5byte
6  servoPinMap[TOTAL_PINS];
7byte servoCount = 0;
8
9void analogWriteCallback(byte
10  pin, int value)
11{
12  if (IS_PIN_DIGITAL(pin)) {
13    servos[servoPinMap[pin]].write(value);
14
15  }
16}
17
18void systemResetCallback()
19{
20  servoCount = 0;
21}
22
23void
24  setup()
25{
26  byte pin;
27
28  Firmata.setFirmwareVersion(FIRMATA_FIRMWARE_MAJOR_VERSION,
29  FIRMATA_FIRMWARE_MINOR_VERSION);
30  Firmata.attach(ANALOG_MESSAGE, analogWriteCallback);
31
32  Firmata.attach(SYSTEM_RESET, systemResetCallback);
33
34  Firmata.begin(57600);
35
36  systemResetCallback();
37
38  // attach servos from first digital pin up to
39  max number of
40  // servos supported for the board
41  for (pin = 0; pin < TOTAL_PINS;
42  pin++) {
43    if (IS_PIN_DIGITAL(pin)) {
44      if (servoCount < MAX_SERVOS)
45  {
46        servoPinMap[pin] = servoCount;
47        servos[servoPinMap[pin]].attach(PIN_TO_DIGITAL(pin));
48
49        servoCount++;
50      }
51    }
52  }
53}
54
55void loop()
56{
57
58  while (Firmata.available())
59    Firmata.processInput();
60}