loading...
  1. Simple Hand/Finger Tracking & Gesture Recognition

Simple Hand/Finger Tracking & Gesture Recognition

Once again, hello !!

In this post, we are going to cover Hand Gesture Tracking. It’s one of my favorite and fun topic. It is often considered as a tough task to perform Hand or finger tracking due to the limitations of the Background scenes. One such condition also exists in this program. You need to have a clear background in order for the program to successful be able to run.

Here is how we are going to perform it:
1) Convert the video frame from BGR to Gray
2) Perform a Threshold OTSU
3) Perform a GBlur
4) Find the Biggest Contour(this will be our hand)
5) Perform a convexHull and mark the ROI
6) Count the Items
7) Display it

Here is how we recognize the gestures/fingers.

If 1 finger on screen –> we’ll display “Single”
If 2 finger on screen –> we’ll display “Victory”
If 3 finger on screen –> we’ll display “Trio Fingers”
If 4 finger on screen –> we’ll display “Four Fingers”
If 5 finger on screen –> we’ll display “High Five”

You can change there text by editing the below code:

1
2
3
4
5
6
7
8
9
10
11
12
                                 if(count==1)
                                strcpy(a,"Single");
                            else if(count==2)
                                strcpy(a,"Victory");
                            else if(count==3)
                                strcpy(a,"Trio Fingers");
                            else if(count==4)
                                strcpy(a,"Four Fingers");
                            else if(count==5)
                                strcpy(a,"High Five");
                            else
                                strcpy(a,"Hello There");

A look at the demo:

Once, again, we’ll start off by opening visual studio 2015 and creating a new project.
Loading all the required OpenCV includes and Dll’s and creating a new main.cpp file.
Copy and paste the below code and you can enjoy hand tracking.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
#include "opencv2/imgproc/imgproc.hpp"
#include <opencv2/highgui/highgui.hpp>
#include <iostream>
 
using namespace cv;
using namespace std;
 
int main( int argc, const char** argv)
{
 
 
    VideoCapture cam(0);
    if(!cam.isOpened()){
        cout<<"ERROR not opened "<< endl;
        return -1;
    }
    Mat img;
    Mat img_threshold;
    Mat img_gray;
    Mat img_roi;
    namedWindow("Original_image",CV_WINDOW_AUTOSIZE);
    namedWindow("Gray_image",CV_WINDOW_AUTOSIZE);
    namedWindow("Thresholded_image",CV_WINDOW_AUTOSIZE);
    namedWindow("ROI",CV_WINDOW_AUTOSIZE);
    char a[40];
    int count =0;
    while(1){
        bool b=cam.read(img);
        if(!b){
            cout<<"ERROR : cannot read"<<endl;
            return -1;
        }
        Rect roi(340,100,270,270);
        img_roi=img(roi);
        cvtColor(img_roi,img_gray,CV_RGB2GRAY);
 
        GaussianBlur(img_gray,img_gray,Size(19,19),0.0,0);
        threshold(img_gray,img_threshold,0,255,THRESH_BINARY_INV+THRESH_OTSU);
 
        vector<vector<Point> >contours;
        vector<Vec4i>hierarchy;
        findContours(img_threshold,contours,hierarchy,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE,Point());
        if(contours.size()>0){
                size_t indexOfBiggestContour = -1;
	            size_t sizeOfBiggestContour = 0;
 
	            for (size_t i = 0; i < contours.size(); i++){
		           if(contours[i].size() > sizeOfBiggestContour){
			       sizeOfBiggestContour = contours[i].size();
			       indexOfBiggestContour = i;
		          }
                }
                vector<vector<int> >hull(contours.size());
                vector<vector<Point> >hullPoint(contours.size());
                vector<vector<Vec4i> >defects(contours.size());
                vector<vector<Point> >defectPoint(contours.size());
                vector<vector<Point> >contours_poly(contours.size());
                Point2f rect_point[4];
                vector<RotatedRect>minRect(contours.size());
                vector<Rect> boundRect(contours.size());
                for(size_t i=0;i<contours.size();i++){
                    if(contourArea(contours[i])>5000){
                        convexHull(contours[i],hull[i],true);
                        convexityDefects(contours[i],hull[i],defects[i]);
                        if(indexOfBiggestContour==i){
                            minRect[i]=minAreaRect(contours[i]);
                            for(size_t k=0;k<hull[i].size();k++){
                                int ind=hull[i][k];
                                hullPoint[i].push_back(contours[i][ind]);
                            }
                            count =0;
 
                            for(size_t k=0;k<defects[i].size();k++){
                                if(defects[i][k][3]>13*256){
                                 /*   int p_start=defects[i][k][0];   */
                                    int p_end=defects[i][k][1];
                                    int p_far=defects[i][k][2];
                                    defectPoint[i].push_back(contours[i][p_far]);
                                    circle(img_roi,contours[i][p_end],3,Scalar(0,255,0),2);
                                    count++;
                                }
 
                            }
 
                              if(count==1)
                                strcpy(a,"Single");
                            else if(count==2)
                                strcpy(a,"Victory");
                            else if(count==3)
                                strcpy(a,"Trio Fingers");
                            else if(count==4)
                                strcpy(a,"Four Fingers");
                            else if(count==5)
                                strcpy(a,"High Five");
                            else
                                strcpy(a,"Hello There");
 
                            putText(img,a,Point(70,70),CV_FONT_HERSHEY_SIMPLEX,3,Scalar(255,0,0),2,8,false);
                            drawContours(img_threshold, contours, i,Scalar(255,255,0),2, 8, vector<Vec4i>(), 0, Point() );
                            drawContours(img_threshold, hullPoint, i, Scalar(255,255,0),1, 8, vector<Vec4i>(),0, Point());
                            drawContours(img_roi, hullPoint, i, Scalar(0,0,255),2, 8, vector<Vec4i>(),0, Point() );
                            approxPolyDP(contours[i],contours_poly[i],3,false);
                            boundRect[i]=boundingRect(contours_poly[i]);
                            rectangle(img_roi,boundRect[i].tl(),boundRect[i].br(),Scalar(255,0,0),2,8,0);
                            minRect[i].points(rect_point);
                            for(size_t k=0;k<4;k++){
                                line(img_roi,rect_point[k],rect_point[(k+1)%4],Scalar(0,255,0),2,8);
                            }
 
                        }
                    }
 
                }
            imshow("Original_image",img);
            imshow("Gray_image",img_gray);
            imshow("Thresholded_image",img_threshold);
            imshow("ROI",img_roi);
            if(waitKey(30)==27){
                  return -1;
             }
 
        }
 
    }
 
     return 0;
}

Happy Coding !!!





2 thoughts on “Simple Hand/Finger Tracking & Gesture Recognition”

  1. Roger says:

    You have been posting really awesome tutorial man. Great job.

    1. Editor says:

      Thank you Roger.

Leave a Reply

Your email address will not be published. Required fields are marked *

Welcome to OpenCV World !! Come as a Guest, stay as a Family