Why I Made This
Being new to coding, I wanted to make something futuristic, fun, and slightly complicated. I chose to make a Hand Gesture Mouse Controller based on Python, OpenCV, and MediaPipe. The notion of being able to control your computer using your hand alone seemed like science fiction, and I was totally in.
TL;DR: It wasn't flawless, but I gained a lot of insight into image processing, Python libraries, and the ability of hand gestures to direct real-world behavior.
What I Used
Python
OpenCV (for video processing)
MediaPipe (for hand detection and landmarks)
PyAutoGUI (for mouse movement and clicking)
pycaw (for volume adjustment)
How It Works
Here's the big-picture logic:
Record webcam input using OpenCV.
Find hand landmarks with MediaPipe.
Track finger locations, such as the thumb, index, and pinky.
Translate the hand movement onto screen coordinates.
Perform click, scroll, or volume gestures.
import cv2
import mediapipe as mp
import pyautogui
import numpy as np
import time
import math
from ctypes import cast, POINTER
from comtypes import CLSCTX_ALL
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume
pyautogui.FAILSAFE = False
wCam, hCam = 640, 480
frameR = 100
smoothening = 6
plocX, plocY = 0, 0
clocX, clocY = 0, 0
click_state = False
scroll_timer = time.time()
screenshot_timer = 0
# Volume control setup
devices = AudioUtilities.GetSpeakers()
interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
volume = cast(interface, POINTER(IAudioEndpointVolume))
vol_min, vol_max = volume.GetVolumeRange()[:2]
cap = cv2.VideoCapture(0)
cap.set(3, wCam)
cap.set(4, hCam)
screen_w, screen_h = pyautogui.size()
mpHands = mp.solutions.hands
hands = mpHands.Hands(max_num_hands=1, min_detection_confidence=0.75)
mpDraw = mp.solutions.drawing_utils
while True:
success, img = cap.read()
img = cv2.flip(img, 1)
imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
results = hands.process(imgRGB)
if results.multi_hand_landmarks:
for handLms in results.multi_hand_landmarks:
lm = handLms.landmark
x1 = int(lm[8].x * wCam)
y1 = int(lm[8].y * hCam)
cx, cy = int(lm[0].x * wCam), int(lm[0].y * hCam)
tips = [8, 12, 16, 20]
fingers = [1 if lm[tip].y < lm[tip - 2].y else 0 for tip in tips]
if fingers == [1, 0, 0, 0]:
x3 = np.interp(x1, (frameR, wCam - frameR), (0, screen_w))
y3 = np.interp(y1, (frameR, hCam - frameR), (0, screen_h))
clocX = plocX + (x3 - plocX) / smoothening
clocY = plocY + (y3 - plocY) / smoothening
pyautogui.moveTo(clocX, clocY)
plocX, plocY = clocX, clocY
thumb_tip = lm[4]
index_tip = lm[8]
dist_click = np.linalg.norm(np.array([thumb_tip.x, thumb_tip.y]) - np.array([index_tip.x, index_tip.y]))
if dist_click < 0.03 and not click_state:
pyautogui.click()
click_state = True
elif dist_click > 0.05:
click_state = False
if fingers[0] == 1 and fingers[1] == 1:
if time.time() - scroll_timer > 0.25:
if lm[8].y < lm[6].y and lm[12].y < lm[10].y:
pyautogui.scroll(-60)
elif lm[8].y > lm[6].y and lm[12].y > lm[10].y:
pyautogui.scroll(60)
scroll_timer = time.time()
if fingers == [0, 0, 0, 0]:
x5, y5 = lm[5].x, lm[5].y
x17, y17 = lm[17].x, lm[17].y
angle = math.degrees(math.atan2(y17 - y5, x17 - x5))
if angle > 30:
volume.SetMasterVolumeLevel(min(vol_max, volume.GetMasterVolumeLevel() + 1.0), None)
elif angle < -30:
volume.SetMasterVolumeLevel(max(vol_min, volume.GetMasterVolumeLevel() - 1.0), None)
if fingers == [1, 1, 1, 1]:
if screenshot_timer == 0:
screenshot_timer = time.time()
elif time.time() - screenshot_timer > 2:
pyautogui.screenshot().save("screenshot.png")
screenshot_timer = 0
else:
screenshot_timer = 0
mpDraw.draw_landmarks(img, handLms, mpHands.HAND_CONNECTIONS)
cv2.imshow("Hand Gesture Controller", img)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()