loading...
  1. Fast Object Tracking based on HSV, YUV, RGB & YCrCb Threshold and Contours Detection with X, Y Coordinates

Fast Object Tracking based on HSV, YUV, RGB & YCrCb Threshold and Contours Detection with X, Y Coordinates

Hello once again,
Happy Wednesday,

In today’s post, we’ll perform the Object Tracking by using the image thresholding and contours detection. In the code you’ll notice that we are creating a trackbar to adjust the HSV values of the object to find out preferred object and then the tracking code will start detecting this object. If you are unsure what HSV values are used for, you can follow this article.
If you would like to hard-code the HSV value, you can do so by changing the following line in code:

1
2
3
inRange(HSV, Scalar(H_MIN, S_MIN, V_MIN), Scalar(H_MAX, S_MAX, V_MAX), threshold);
#change it to the trackbar values
inRange(HSV, Scalar(50, 50, 100), Scalar(80, 95, 210), threshold);

Infact, you can also change the threshold from HSV to YUV or YCrCb colorspace depending upon your requirements. Just change the below line for that.

1
2
3
4
5
cvtColor(cameraFeed, HSV, COLOR_BGR2HSV);
#change it to the YUV colorspace
cvtColor(cameraFeed, HSV, COLOR_BGR2YUV);
#OR change it to the YCrCb colorspace
cvtColor(cameraFeed, HSV, COLOR_BGR2YCrCb);

Here is how it will look after the successful run. I am detecting a screwdriver based on the HSV colorspace.


All the comments are already present with the code, go through them and you’ll understand what we are doing at each and every step.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
//Code from PythonOpenCV.com
#include <sstream>
#include <string>
#include <iostream>
#include "opencv\highgui.h"
#include "opencv\cv.h"
 
using namespace cv;
using namespace std;
//initial min and max HSV filter values.
//these will be changed using trackbars
int H_MIN = 0;
int H_MAX = 256;
int S_MIN = 0;
int S_MAX = 256;
int V_MIN = 0;
int V_MAX = 256;
//default capture width and height
const int FRAME_WIDTH = 640;
const int FRAME_HEIGHT = 480;
//max number of objects to be detected in frame
const int MAX_NUM_OBJECTS = 50;
//minimum and maximum object area
const int MIN_OBJECT_AREA = 20 * 20;
const int MAX_OBJECT_AREA = FRAME_HEIGHT*FRAME_WIDTH / 1.5;
//names that will appear at the top of each window
const string windowName = "Original Image";
const string windowName1 = "HSV Image";
const string windowName2 = "Thresholded Image";
const string windowName3 = "After Morphological Operations";
const string trackbarWindowName = "Trackbars";
 
bool calibrationMode;//used for showing debugging windows, trackbars etc.
 
bool mouseIsDragging;//used for showing a rectangle on screen as user clicks and drags mouse
bool mouseMove;
bool rectangleSelected;
cv::Point initialClickPoint, currentMousePoint; //keep track of initial point clicked and current position of mouse
cv::Rect rectangleROI; //this is the ROI that the user has selected
vector H_ROI, S_ROI, V_ROI;// HSV values from the click/drag ROI region stored in separate vectors so that we can sort them easily
 
void on_trackbar(int, void*)
{//This function gets called whenever a
	// trackbar position is changed
 
	//for now, this does nothing.
 
 
 
}
void createTrackbars(){
	//create window for trackbars
 
 
	namedWindow(trackbarWindowName, 0);
	//create memory to store trackbar name on window
	char TrackbarName[50];
	sprintf(TrackbarName, "H_MIN", H_MIN);
	sprintf(TrackbarName, "H_MAX", H_MAX);
	sprintf(TrackbarName, "S_MIN", S_MIN);
	sprintf(TrackbarName, "S_MAX", S_MAX);
	sprintf(TrackbarName, "V_MIN", V_MIN);
	sprintf(TrackbarName, "V_MAX", V_MAX);
	//create trackbars and insert them into window
	//3 parameters are: the address of the variable that is changing when the trackbar is moved(eg.H_LOW),
	//the max value the trackbar can move (eg. H_HIGH), 
	//and the function that is called whenever the trackbar is moved(eg. on_trackbar)
	//                                  ----&gt;    ----&gt;     ----&gt;      
	createTrackbar("H_MIN", trackbarWindowName, &amp;H_MIN, 255, on_trackbar);
	createTrackbar("H_MAX", trackbarWindowName, &amp;H_MAX, 255, on_trackbar);
	createTrackbar("S_MIN", trackbarWindowName, &amp;S_MIN, 255, on_trackbar);
	createTrackbar("S_MAX", trackbarWindowName, &amp;S_MAX, 255, on_trackbar);
	createTrackbar("V_MIN", trackbarWindowName, &amp;V_MIN, 255, on_trackbar);
	createTrackbar("V_MAX", trackbarWindowName, &amp;V_MAX, 255, on_trackbar);
 
 
}
void clickAndDrag_Rectangle(int event, int x, int y, int flags, void* param){
	//only if calibration mode is true will we use the mouse to change HSV values
	if (calibrationMode == true){
		//get handle to video feed passed in as "param" and cast as Mat pointer
		Mat* videoFeed = (Mat*)param;
 
		if (event == CV_EVENT_LBUTTONDOWN &amp;&amp; mouseIsDragging == false)
		{
			//keep track of initial point clicked
			initialClickPoint = cv::Point(x, y);
			//user has begun dragging the mouse
			mouseIsDragging = true;
		}
		/* user is dragging the mouse */
		if (event == CV_EVENT_MOUSEMOVE &amp;&amp; mouseIsDragging == true)
		{
			//keep track of current mouse point
			currentMousePoint = cv::Point(x, y);
			//user has moved the mouse while clicking and dragging
			mouseMove = true;
		}
		/* user has released left button */
		if (event == CV_EVENT_LBUTTONUP &amp;&amp; mouseIsDragging == true)
		{
			//set rectangle ROI to the rectangle that the user has selected
			rectangleROI = Rect(initialClickPoint, currentMousePoint);
 
			//reset boolean variables
			mouseIsDragging = false;
			mouseMove = false;
			rectangleSelected = true;
		}
 
		if (event == CV_EVENT_RBUTTONDOWN){
			//user has clicked right mouse button
			//Reset HSV Values
			H_MIN = 0;
			S_MIN = 0;
			V_MIN = 0;
			H_MAX = 255;
			S_MAX = 255;
			V_MAX = 255;
 
		}
		if (event == CV_EVENT_MBUTTONDOWN){
 
			//user has clicked middle mouse button
			//enter code here if needed.
		}
	}
 
}
void recordHSV_Values(cv::Mat frame, cv::Mat hsv_frame){
 
	//save HSV values for ROI that user selected to a vector
	if (mouseMove == false &amp;&amp; rectangleSelected == true){
 
		//clear previous vector values
		if (H_ROI.size()&gt;0) H_ROI.clear();
		if (S_ROI.size()&gt;0) S_ROI.clear();
		if (V_ROI.size()&gt;0 )V_ROI.clear();
		//if the rectangle has no width or height (user has only dragged a line) then we don't try to iterate over the width or height
		if (rectangleROI.width&lt;1 || rectangleROI.height&lt;1) cout &lt;&lt; "Please drag a rectangle, not a line" &lt;&lt; endl;
		else{
			for (int i = rectangleROI.x; i(j, i)[0]);
					S_ROI.push_back((int)hsv_frame.at(j, i)[1]);
					V_ROI.push_back((int)hsv_frame.at(j, i)[2]);
				}
			}
		}
		//reset rectangleSelected so user can select another region if necessary
		rectangleSelected = false;
		//set min and max HSV values from min and max elements of each array
 
		if (H_ROI.size()&gt;0){
			//NOTE: min_element and max_element return iterators so we must dereference them with "*"
			H_MIN = *std::min_element(H_ROI.begin(), H_ROI.end());
			H_MAX = *std::max_element(H_ROI.begin(), H_ROI.end());
			cout &lt;&lt; "MIN 'H' VALUE: " &lt;&lt; H_MIN &lt;&lt; endl;
			cout &lt;&lt; "MAX 'H' VALUE: " &lt;&lt; H_MAX &lt;&lt; endl; } if (S_ROI.size()&gt;0){
			S_MIN = *std::min_element(S_ROI.begin(), S_ROI.end());
			S_MAX = *std::max_element(S_ROI.begin(), S_ROI.end());
			cout &lt;&lt; "MIN 'S' VALUE: " &lt;&lt; S_MIN &lt;&lt; endl;
			cout &lt;&lt; "MAX 'S' VALUE: " &lt;&lt; S_MAX &lt;&lt; endl; } if (V_ROI.size()&gt;0){
			V_MIN = *std::min_element(V_ROI.begin(), V_ROI.end());
			V_MAX = *std::max_element(V_ROI.begin(), V_ROI.end());
			cout &lt;&lt; "MIN 'V' VALUE: " &lt;&lt; V_MIN &lt;&lt; endl;
			cout &lt;&lt; "MAX 'V' VALUE: " &lt;&lt; V_MAX &lt;&lt; endl;
		}
 
	}
 
	if (mouseMove == true){
		//if the mouse is held down, we will draw the click and dragged rectangle to the screen
		rectangle(frame, initialClickPoint, cv::Point(currentMousePoint.x, currentMousePoint.y), cv::Scalar(0, 255, 0), 1, 8, 0);
	}
 
 
}
string intToString(int number){
 
 
	std::stringstream ss;
	ss &lt;&lt; number; return ss.str(); } void drawObject(int x, int y, Mat &amp;frame){ //use some of the openCV drawing functions to draw crosshairs //on your tracked image! //'if' and 'else' statements to prevent //memory errors from writing off the screen (ie. (-25,-25) is not within the window) circle(frame, Point(x, y), 20, Scalar(0, 255, 0), 2); if (y - 25&gt;0)
		line(frame, Point(x, y), Point(x, y - 25), Scalar(0, 255, 0), 2);
	else line(frame, Point(x, y), Point(x, 0), Scalar(0, 255, 0), 2);
	if (y + 25&lt;FRAME_HEIGHT) line(frame, Point(x, y), Point(x, y + 25), Scalar(0, 255, 0), 2); else line(frame, Point(x, y), Point(x, FRAME_HEIGHT), Scalar(0, 255, 0), 2); if (x - 25&gt;0)
		line(frame, Point(x, y), Point(x - 25, y), Scalar(0, 255, 0), 2);
	else line(frame, Point(x, y), Point(0, y), Scalar(0, 255, 0), 2);
	if (x + 25&lt;FRAME_WIDTH)
		line(frame, Point(x, y), Point(x + 25, y), Scalar(0, 255, 0), 2);
	else line(frame, Point(x, y), Point(FRAME_WIDTH, y), Scalar(0, 255, 0), 2);
 
	putText(frame, intToString(x) + "," + intToString(y), Point(x, y + 30), 1, 1, Scalar(0, 255, 0), 2);
 
}
void morphOps(Mat &amp;thresh){
 
	//create structuring element that will be used to "dilate" and "erode" image.
	//the element chosen here is a 3px by 3px rectangle
 
	Mat erodeElement = getStructuringElement(MORPH_RECT, Size(3, 3));
	//dilate with larger element so make sure object is nicely visible
	Mat dilateElement = getStructuringElement(MORPH_RECT, Size(8, 8));
 
	erode(thresh, thresh, erodeElement);
	erode(thresh, thresh, erodeElement);
 
 
	dilate(thresh, thresh, dilateElement);
	dilate(thresh, thresh, dilateElement);
 
 
 
}
void trackFilteredObject(int &amp;x, int &amp;y, Mat threshold, Mat &amp;cameraFeed){
 
	Mat temp;
	threshold.copyTo(temp);
	//these two vectors needed for output of findContours
	vector&lt; vector &gt; contours;
	vector hierarchy;
	//find contours of filtered image using openCV findContours function
	findContours(temp, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
	//use moments method to find our filtered object
	double refArea = 0;
	int largestIndex = 0;
	bool objectFound = false;
	if (hierarchy.size() &gt; 0) {
		int numObjects = hierarchy.size();
		//if number of objects greater than MAX_NUM_OBJECTS we have a noisy filter
		if (numObjects&lt;MAX_NUM_OBJECTS){ for (int index = 0; index &gt;= 0; index = hierarchy[index][0]) {
 
				Moments moment = moments((cv::Mat)contours[index]);
				double area = moment.m00;
 
				//if the area is less than 20 px by 20px then it is probably just noise
				//if the area is the same as the 3/2 of the image size, probably just a bad filter
				//we only want the object with the largest area so we save a reference area each
				//iteration and compare it to the area in the next iteration.
				if (area&gt;MIN_OBJECT_AREA &amp;&amp; arearefArea){
					x = moment.m10 / area;
					y = moment.m01 / area;
					objectFound = true;
					refArea = area;
					//save index of largest contour to use with drawContours
					largestIndex = index;
				}
				else objectFound = false;
 
 
			}
			//let user know you found an object
			if (objectFound == true){
				putText(cameraFeed, "Tracking Object", Point(0, 50), 2, 1, Scalar(0, 255, 0), 2);
				//draw object location on screen
				drawObject(x, y, cameraFeed);
				//draw largest contour
				//drawContours(cameraFeed, contours, largestIndex, Scalar(0, 255, 255), 2);
			}
 
		}
		else putText(cameraFeed, "TOO MUCH NOISE! ADJUST FILTER", Point(0, 50), 1, 2, Scalar(0, 0, 255), 2);
	}
}
int main(int argc, char* argv[])
{
	//some boolean variables for different functionality within this
	//program
	bool trackObjects = true;
	bool useMorphOps = true;
	calibrationMode = true;
	//Matrix to store each frame of the webcam feed
	Mat cameraFeed;
	//matrix storage for HSV image
	Mat HSV;
	//matrix storage for binary threshold image
	Mat threshold;
	//x and y values for the location of the object
	int x = 0, y = 0;
	//video capture object to acquire webcam feed
	VideoCapture capture;
	//open capture object at location zero (default location for webcam)
	capture.open(0);
	//set height and width of capture frame
	capture.set(CV_CAP_PROP_FRAME_WIDTH, FRAME_WIDTH);
	capture.set(CV_CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT);
	//must create a window before setting mouse callback
	cv::namedWindow(windowName);
	//set mouse callback function to be active on "Webcam Feed" window
	//we pass the handle to our "frame" matrix so that we can draw a rectangle to it
	//as the user clicks and drags the mouse
	cv::setMouseCallback(windowName, clickAndDrag_Rectangle, &amp;cameraFeed);
	//initiate mouse move and drag to false 
	mouseIsDragging = false;
	mouseMove = false;
	rectangleSelected = false;
 
	//start an infinite loop where webcam feed is copied to cameraFeed matrix
	//all of our operations will be performed within this loop
	while (1){
		//store image to matrix
		capture.read(cameraFeed);
		//convert frame from BGR to HSV colorspace
		cvtColor(cameraFeed, HSV, COLOR_BGR2HSV);
		//set HSV values from user selected region
		recordHSV_Values(cameraFeed, HSV);
		//filter HSV image between values and store filtered image to
		//threshold matrix
		inRange(HSV, Scalar(H_MIN, S_MIN, V_MIN), Scalar(H_MAX, S_MAX, V_MAX), threshold);
		//perform morphological operations on thresholded image to eliminate noise
		//and emphasize the filtered object(s)
		if (useMorphOps)
			morphOps(threshold);
		//pass in thresholded frame to our object tracking function
		//this function will return the x and y coordinates of the
		//filtered object
		if (trackObjects)
			trackFilteredObject(x, y, threshold, cameraFeed);
 
		//show frames 
		if (calibrationMode == true){
 
			//create slider bars for HSV filtering
			createTrackbars();
			imshow(windowName1, HSV);
			imshow(windowName2, threshold);
		}
		else{
 
			destroyWindow(windowName1);
			destroyWindow(windowName2);
			destroyWindow(trackbarWindowName);
		}
		imshow(windowName, cameraFeed);
 
 
 
		//delay 30ms so that screen can refresh.
		//image will not appear without this waitKey() command
		//also use waitKey command to capture keyboard input
		if (waitKey(30) == 99) calibrationMode = !calibrationMode;//if user presses 'c', toggle calibration mode
	}
 
 
 
 
 
 
	return 0;
}

Happy Coding.





Leave a Reply

Your email address will not be published. Required fields are marked *

Welcome to OpenCV World !! Come as a Guest, stay as a Family