Simple Hand/Finger Tracking & Gesture Recognition
Once again, hello !!
In this post, we are going to cover Hand Gesture Tracking. It’s one of my favorite and fun topic. It is often considered as a tough task to perform Hand or finger tracking due to the limitations of the Background scenes. One such condition also exists in this program. You need to have a clear background in order for the program to successful be able to run.
Here is how we are going to perform it:
1) Convert the video frame from BGR to Gray
2) Perform a Threshold OTSU
3) Perform a GBlur
4) Find the Biggest Contour(this will be our hand)
5) Perform a convexHull and mark the ROI
6) Count the Items
7) Display it
Here is how we recognize the gestures/fingers.
If 1 finger on screen –> we’ll display “Single”
If 2 finger on screen –> we’ll display “Victory”
If 3 finger on screen –> we’ll display “Trio Fingers”
If 4 finger on screen –> we’ll display “Four Fingers”
If 5 finger on screen –> we’ll display “High Five”
You can change there text by editing the below code:
1 2 3 4 5 6 7 8 9 10 11 12 | if(count==1) strcpy(a,"Single"); else if(count==2) strcpy(a,"Victory"); else if(count==3) strcpy(a,"Trio Fingers"); else if(count==4) strcpy(a,"Four Fingers"); else if(count==5) strcpy(a,"High Five"); else strcpy(a,"Hello There"); |
A look at the demo:
Once, again, we’ll start off by opening visual studio 2015 and creating a new project.
Loading all the required OpenCV includes and Dll’s and creating a new main.cpp file.
Copy and paste the below code and you can enjoy hand tracking.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | #include "opencv2/imgproc/imgproc.hpp" #include <opencv2/highgui/highgui.hpp> #include <iostream> using namespace cv; using namespace std; int main( int argc, const char** argv) { VideoCapture cam(0); if(!cam.isOpened()){ cout<<"ERROR not opened "<< endl; return -1; } Mat img; Mat img_threshold; Mat img_gray; Mat img_roi; namedWindow("Original_image",CV_WINDOW_AUTOSIZE); namedWindow("Gray_image",CV_WINDOW_AUTOSIZE); namedWindow("Thresholded_image",CV_WINDOW_AUTOSIZE); namedWindow("ROI",CV_WINDOW_AUTOSIZE); char a[40]; int count =0; while(1){ bool b=cam.read(img); if(!b){ cout<<"ERROR : cannot read"<<endl; return -1; } Rect roi(340,100,270,270); img_roi=img(roi); cvtColor(img_roi,img_gray,CV_RGB2GRAY); GaussianBlur(img_gray,img_gray,Size(19,19),0.0,0); threshold(img_gray,img_threshold,0,255,THRESH_BINARY_INV+THRESH_OTSU); vector<vector<Point> >contours; vector<Vec4i>hierarchy; findContours(img_threshold,contours,hierarchy,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE,Point()); if(contours.size()>0){ size_t indexOfBiggestContour = -1; size_t sizeOfBiggestContour = 0; for (size_t i = 0; i < contours.size(); i++){ if(contours[i].size() > sizeOfBiggestContour){ sizeOfBiggestContour = contours[i].size(); indexOfBiggestContour = i; } } vector<vector<int> >hull(contours.size()); vector<vector<Point> >hullPoint(contours.size()); vector<vector<Vec4i> >defects(contours.size()); vector<vector<Point> >defectPoint(contours.size()); vector<vector<Point> >contours_poly(contours.size()); Point2f rect_point[4]; vector<RotatedRect>minRect(contours.size()); vector<Rect> boundRect(contours.size()); for(size_t i=0;i<contours.size();i++){ if(contourArea(contours[i])>5000){ convexHull(contours[i],hull[i],true); convexityDefects(contours[i],hull[i],defects[i]); if(indexOfBiggestContour==i){ minRect[i]=minAreaRect(contours[i]); for(size_t k=0;k<hull[i].size();k++){ int ind=hull[i][k]; hullPoint[i].push_back(contours[i][ind]); } count =0; for(size_t k=0;k<defects[i].size();k++){ if(defects[i][k][3]>13*256){ /* int p_start=defects[i][k][0]; */ int p_end=defects[i][k][1]; int p_far=defects[i][k][2]; defectPoint[i].push_back(contours[i][p_far]); circle(img_roi,contours[i][p_end],3,Scalar(0,255,0),2); count++; } } if(count==1) strcpy(a,"Single"); else if(count==2) strcpy(a,"Victory"); else if(count==3) strcpy(a,"Trio Fingers"); else if(count==4) strcpy(a,"Four Fingers"); else if(count==5) strcpy(a,"High Five"); else strcpy(a,"Hello There"); putText(img,a,Point(70,70),CV_FONT_HERSHEY_SIMPLEX,3,Scalar(255,0,0),2,8,false); drawContours(img_threshold, contours, i,Scalar(255,255,0),2, 8, vector<Vec4i>(), 0, Point() ); drawContours(img_threshold, hullPoint, i, Scalar(255,255,0),1, 8, vector<Vec4i>(),0, Point()); drawContours(img_roi, hullPoint, i, Scalar(0,0,255),2, 8, vector<Vec4i>(),0, Point() ); approxPolyDP(contours[i],contours_poly[i],3,false); boundRect[i]=boundingRect(contours_poly[i]); rectangle(img_roi,boundRect[i].tl(),boundRect[i].br(),Scalar(255,0,0),2,8,0); minRect[i].points(rect_point); for(size_t k=0;k<4;k++){ line(img_roi,rect_point[k],rect_point[(k+1)%4],Scalar(0,255,0),2,8); } } } } imshow("Original_image",img); imshow("Gray_image",img_gray); imshow("Thresholded_image",img_threshold); imshow("ROI",img_roi); if(waitKey(30)==27){ return -1; } } } return 0; } |
Happy Coding !!!
You have been posting really awesome tutorial man. Great job.
Thank you Roger.