Update the trunk to the OpenCV's CVS (2008-07-14)
[opencv] / cvaux / src / vs / blobtrackingmsfg.cpp
1 /*M///////////////////////////////////////////////////////////////////////////////////////
2 //
3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4 //
5 //  By downloading, copying, installing or using the software you agree to this license.
6 //  If you do not agree to this license, do not download, install,
7 //  copy or use the software.
8 //
9 //
10 //                        Intel License Agreement
11 //
12 // Copyright (C) 2000, Intel Corporation, all rights reserved.
13 // Third party copyrights are property of their respective owners.
14 //
15 // Redistribution and use in source and binary forms, with or without modification,
16 // are permitted provided that the following conditions are met:
17 //
18 //   * Redistribution's of source code must retain the above copyright notice,
19 //     this list of conditions and the following disclaimer.
20 //
21 //   * Redistribution's in binary form must reproduce the above copyright notice,
22 //     this list of conditions and the following disclaimer in the documentation
23 //     and/or other materials provided with the distribution.
24 //
25 //   * The name of Intel Corporation may not be used to endorse or promote products
26 //     derived from this software without specific prior written permission.
27 //
28 // This software is provided by the copyright holders and contributors "as is" and
29 // any express or implied warranties, including, but not limited to, the implied
30 // warranties of merchantability and fitness for a particular purpose are disclaimed.
31 // In no event shall the Intel Corporation or contributors be liable for any direct,
32 // indirect, incidental, special, exemplary, or consequential damages
33 // (including, but not limited to, procurement of substitute goods or services;
34 // loss of use, data, or profits; or business interruption) however caused
35 // and on any theory of liability, whether in contract, strict liability,
36 // or tort (including negligence or otherwise) arising in any way out of
37 // the use of this software, even if advised of the possibility of such damage.
38 //
39 //M*/
40
41 #include "_cvaux.h"
42
43 // Uncomment to trade flexibility for speed
44 //#define CONST_HIST_SIZE
45
46 // Uncomment to get some performance stats in stderr
47 //#define REPORT_TICKS
48
49 #ifdef CONST_HIST_SIZE
50 #define m_BinBit 5
51 #define m_ByteShift 3
52 #endif
53
54 typedef float DefHistType;
55 #define DefHistTypeMat CV_32F
56 #define HIST_INDEX(_pData) (((_pData)[0]>>m_ByteShift) + (((_pData)[1]>>(m_ByteShift))<<m_BinBit)+((pImgData[2]>>m_ByteShift)<<(m_BinBit*2)))
57
58 class DefHist
59 {
60 public:
61     CvMat*          m_pHist;
62     DefHistType     m_HistVolume;
63     DefHist(int BinNum=0)
64     {
65         m_pHist = NULL;
66         m_HistVolume = 0;
67         Resize(BinNum);
68     }
69
70     ~DefHist()
71     {
72         if(m_pHist)cvReleaseMat(&m_pHist);
73     }
74
75     void Resize(int BinNum)
76     {
77         if(m_pHist)cvReleaseMat(&m_pHist);
78         if(BinNum>0)
79         {
80             m_pHist = cvCreateMat(1, BinNum, DefHistTypeMat);
81             cvZero(m_pHist);
82         }
83         m_HistVolume = 0;
84     }
85
86     void Update(DefHist* pH, float W)
87     {   /* Update histogram: */
88         double  Vol, WM, WC;
89         Vol = 0.5*(m_HistVolume + pH->m_HistVolume);
90         WM = Vol*(1-W)/m_HistVolume;
91         WC = Vol*(W)/pH->m_HistVolume;
92         cvAddWeighted(m_pHist, WM, pH->m_pHist,WC,0,m_pHist);
93         m_HistVolume = (float)cvSum(m_pHist).val[0];
94     }   /* Update histogram: */
95 };
96
97 class CvBlobTrackerOneMSFG:public CvBlobTrackerOne
98 {
99 protected:
100     int             m_BinNumTotal; /* protected only for parralel MSPF */
101     CvSize          m_ObjSize;
102
103     void ReAllocKernel(int  w, int h)
104     {
105         int     x,y;
106         float   x0 = 0.5f*(w-1);
107         float   y0 = 0.5f*(h-1);
108         assert(w>0);
109         assert(h>0);
110         m_ObjSize = cvSize(w,h);
111
112         if(m_KernelHist) cvReleaseMat(&m_KernelHist);
113         if(m_KernelMeanShift) cvReleaseMat(&m_KernelMeanShift);
114         m_KernelHist = cvCreateMat(h, w, DefHistTypeMat);
115         m_KernelMeanShift = cvCreateMat(h, w, DefHistTypeMat);
116
117         for(y=0; y<h; ++y) for(x=0; x<w; ++x)
118         {
119             double r2 = ((x-x0)*(x-x0)/(x0*x0)+(y-y0)*(y-y0)/(y0*y0));
120 //            double r2 = ((x-x0)*(x-x0)+(y-y0)*(y-y0))/((y0*y0)+(x0*x0));
121             CV_MAT_ELEM(m_KernelHist[0],DefHistType, y, x) = (DefHistType)GetKernelHist(r2);
122             CV_MAT_ELEM(m_KernelMeanShift[0],DefHistType, y, x) = (DefHistType)GetKernelMeanShift(r2);
123         }
124     }
125
126 private:
127     /* Parameters: */
128     int             m_IterNum;
129     float           m_FGWeight;
130     float           m_Alpha;
131     CvMat*          m_KernelHist;
132     CvMat*          m_KernelMeanShift;
133 #ifndef CONST_HIST_SIZE
134     int             m_BinBit;
135     int             m_ByteShift;
136 #endif
137     int             m_BinNum;
138     int             m_Dim;
139     /*
140     CvMat*          m_HistModel;
141     float           m_HistModelVolume;
142     CvMat*          m_HistCandidate;
143     float           m_HistCandidateVolume;
144     CvMat*          m_HistTemp;
145     */
146     DefHist         m_HistModel;
147     DefHist         m_HistCandidate;
148     DefHist         m_HistTemp;
149
150     CvBlob          m_Blob;
151     int             m_Collision;
152
153     void ReAllocHist(int Dim, int BinBit)
154     {
155 #ifndef CONST_HIST_SIZE
156         m_BinBit = BinBit;
157         m_ByteShift = 8-BinBit;
158 #endif
159         m_Dim = Dim;
160         m_BinNum = (1<<BinBit);
161         m_BinNumTotal = cvRound(pow((double)m_BinNum,(double)m_Dim));
162         /*
163         if(m_HistModel) cvReleaseMat(&m_HistModel);
164         if(m_HistCandidate) cvReleaseMat(&m_HistCandidate);
165         if(m_HistTemp) cvReleaseMat(&m_HistTemp);
166         m_HistCandidate = cvCreateMat(1, m_BinNumTotal, DefHistTypeMat);
167         m_HistModel = cvCreateMat(1, m_BinNumTotal, DefHistTypeMat);
168         m_HistTemp = cvCreateMat(1, m_BinNumTotal, DefHistTypeMat);
169         cvZero(m_HistCandidate);
170         cvZero(m_HistModel);
171         m_HistModelVolume = 0.0f;
172         m_HistCandidateVolume = 0.0f;
173         */
174         m_HistCandidate.Resize(m_BinNumTotal);
175         m_HistModel.Resize(m_BinNumTotal);
176         m_HistTemp.Resize(m_BinNumTotal);
177     }
178
179     double GetKernelHist(double r2)
180     {
181         return (r2 < 1) ? 1 -  r2 : 0;
182     }
183
184     double GetKernelMeanShift(double r2)
185     {
186         return (r2<1) ? 1 : 0;
187     }
188
189 //    void CollectHist(IplImage* pImg, IplImage* pMask, CvPoint Center, CvMat* pHist, DefHistType* pHistVolume)
190 //    void CollectHist(IplImage* pImg, IplImage* pMask, CvPoint Center, DefHist* pHist)
191
192     void CollectHist(IplImage* pImg, IplImage* pMask, CvBlob* pBlob, DefHist* pHist)
193     {
194         int UsePrecalculatedKernel = 0;
195         int BW = cvRound(pBlob->w);
196         int BH = cvRound(pBlob->h);
197         DefHistType Volume = 0;
198         int         x0 = cvRound(pBlob->x - BW*0.5);
199         int         y0 = cvRound(pBlob->y - BH*0.5);
200         int         x,y;
201
202         UsePrecalculatedKernel = (BW == m_ObjSize.width && BH == m_ObjSize.height ) ;
203
204         //cvZero(pHist);
205         cvSet(pHist->m_pHist,cvScalar(1.0/m_BinNumTotal)); /* no zero bins, all bins have very small value*/
206         Volume = 1;
207
208         assert(BW < pImg->width);
209         assert(BH < pImg->height);
210         if((x0+BW)>=pImg->width) BW=pImg->width-x0-1;
211         if((y0+BH)>=pImg->height) BH=pImg->height-y0-1;
212         if(x0<0){ x0=0;}
213         if(y0<0){ y0=0;}
214
215         if(m_Dim == 3)
216         {
217             for(y=0; y<BH; ++y)
218             {
219                 unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y+y0,x0*3);
220                 unsigned char* pMaskData = pMask?(&CV_IMAGE_ELEM(pMask,unsigned char,y+y0,x0)):NULL;
221                 DefHistType* pKernelData = NULL;
222
223                 if(UsePrecalculatedKernel)
224                 {
225                     pKernelData = ((DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelHist[0],y,0,sizeof(DefHistType)));
226                 }
227
228                 for(x=0; x<BW; ++x, pImgData+=3)
229                 {
230                     DefHistType K;
231                     int index = HIST_INDEX(pImgData);
232                     assert(index >= 0 && index < pHist->m_pHist->cols);
233
234                     if(UsePrecalculatedKernel)
235                     {
236                         K = pKernelData[x];
237                     }
238                     else
239                     {
240                         float dx = (x+x0-pBlob->x)/(pBlob->w*0.5f);
241                         float dy = (y+y0-pBlob->y)/(pBlob->h*0.5f);
242                         double r2 = dx*dx+dy*dy;
243                         K = (float)GetKernelHist(r2);
244                     }
245
246                     if(pMaskData)
247                     {
248                         K *= pMaskData[x]*0.003921568627450980392156862745098f;
249                     }
250                     Volume += K;
251                     ((DefHistType*)(pHist->m_pHist->data.ptr))[index] += K;
252
253                 }   /* Next column. */
254             }   /*  Next row. */
255         }   /* if m_Dim == 3. */
256
257         pHist->m_HistVolume = Volume;
258
259     };  /* CollectHist */
260
261     double calcBhattacharyya(DefHist* pHM = NULL, DefHist* pHC = NULL, DefHist* pHT = NULL)
262     {
263         if(pHM==NULL) pHM = &m_HistModel;
264         if(pHC==NULL) pHC = &m_HistCandidate;
265         if(pHT==NULL) pHT = &m_HistTemp;
266         if(pHC->m_HistVolume*pHM->m_HistVolume > 0)
267         {
268 #if 0
269             // Use CV functions:
270             cvMul(pHM->m_pHist,pHC->m_pHist,pHT->m_pHist);
271             cvPow(pHT->m_pHist,pHT->m_pHist,0.5);
272             return cvSum(pHT->m_pHist).val[0] / sqrt(pHC->m_HistVolume*pHM->m_HistVolume);
273 #else
274             // Do computations manually and let autovectorizer do the job:
275             DefHistType *hm, *hc, *ht;
276             double sum;
277             int     size;
278             hm=(DefHistType *)(pHM->m_pHist->data.ptr);
279             hc=(DefHistType *)(pHC->m_pHist->data.ptr);
280             ht=(DefHistType *)(pHT->m_pHist->data.ptr);
281             size = pHM->m_pHist->width*pHM->m_pHist->height;
282             sum = 0.;
283             for(int i = 0; i < size; i++ )
284             {
285                 sum += sqrt(hm[i]*hc[i]);
286             }
287             return sum / sqrt(pHC->m_HistVolume*pHM->m_HistVolume);
288 #endif
289         }
290         return 0;
291     }   /* calcBhattacharyyaCoefficient. */
292
293 protected:
294     // double GetBhattacharyya(IplImage* pImg, IplImage* pImgFG, float x, float y, DefHist* pHist=NULL)
295     double GetBhattacharyya(IplImage* pImg, IplImage* pImgFG, CvBlob* pBlob, DefHist* pHist=NULL, int /*thread_number*/ = 0)
296     {
297         if(pHist==NULL)pHist = &m_HistTemp;
298         CollectHist(pImg, pImgFG, pBlob, pHist);
299         return calcBhattacharyya(&m_HistModel, pHist, pHist);
300     }
301
302     void UpdateModelHist(IplImage* pImg, IplImage* pImgFG, CvBlob* pBlob)
303     {
304         if(m_Alpha>0 && !m_Collision)
305         {   /* Update histogram: */
306             CollectHist(pImg, pImgFG, pBlob, &m_HistCandidate);
307             m_HistModel.Update(&m_HistCandidate, m_Alpha);
308         }   /* Update histogram. */
309
310     }   /* UpdateModelHist */
311
312 public:
313     CvBlobTrackerOneMSFG()
314     {
315
316         /* Add several parameters for external use: */
317         m_FGWeight = 2;
318         AddParam("FGWeight", &m_FGWeight);
319         CommentParam("FGWeight","Weight of FG mask using (0 - mask will not be used for tracking)");
320
321         m_Alpha = 0.01f;
322         AddParam("Alpha", &m_Alpha);
323         CommentParam("Alpha","Coefficient for model histogram updating (0 - hist is not upated)");
324
325         m_IterNum = 10;
326         AddParam("IterNum", &m_IterNum);
327         CommentParam("IterNum","Maximal number of iteration in meanshift operation");
328
329         /* Initialize internal data: */
330         m_Collision = 0;
331 //        m_BinBit=0;
332         m_Dim = 0;
333         /*
334         m_HistModel = NULL;
335         m_HistCandidate = NULL;
336         m_HistTemp = NULL;
337         */
338         m_KernelHist = NULL;
339         m_KernelMeanShift = NULL;
340         ReAllocHist(3,5);   /* 3D hist, each dim has 2^5 bins*/
341     }
342
343     ~CvBlobTrackerOneMSFG()
344     {
345         /*
346         if(m_HistModel) cvReleaseMat(&m_HistModel);
347         if(m_HistCandidate) cvReleaseMat(&m_HistCandidate);
348         if(m_HistTemp) cvReleaseMat(&m_HistTemp);
349         */
350         if(m_KernelHist) cvReleaseMat(&m_KernelHist);
351         if(m_KernelMeanShift) cvReleaseMat(&m_KernelMeanShift);
352     }
353
354     /* Interface: */
355     virtual void Init(CvBlob* pBlobInit, IplImage* pImg, IplImage* pImgFG = NULL)
356     {
357         int w = cvRound(CV_BLOB_WX(pBlobInit));
358         int h = cvRound(CV_BLOB_WY(pBlobInit));
359         if(w<CV_BLOB_MINW)w=CV_BLOB_MINW;
360         if(h<CV_BLOB_MINH)h=CV_BLOB_MINH;
361         if(pImg)
362         {
363             if(w>pImg->width)w=pImg->width;
364             if(h>pImg->height)h=pImg->height;
365         }
366         ReAllocKernel(w,h);
367         if(pImg)
368             CollectHist(pImg, pImgFG, pBlobInit, &m_HistModel);
369         m_Blob = pBlobInit[0];
370     };
371
372     virtual CvBlob* Process(CvBlob* pBlobPrev, IplImage* pImg, IplImage* pImgFG = NULL)
373     {
374         int     iter;
375
376         if(pBlobPrev)
377         {
378             m_Blob = pBlobPrev[0];
379         }
380
381         {   /* Check blob size and realloc kernels if it is necessary: */
382             int w = cvRound(m_Blob.w);
383             int h = cvRound(m_Blob.h);
384             if( w != m_ObjSize.width || h!=m_ObjSize.height)
385             {
386                 ReAllocKernel(w,h);
387                 /* after this ( w != m_ObjSize.width || h!=m_ObjSize.height) shoiuld be false */
388             }
389         }   /* Check blob size and realloc kernels if it is necessary: */
390
391
392         for(iter=0; iter<m_IterNum; ++iter)
393         {
394             float   newx=0,newy=0,sum=0;
395             //int     x,y;
396             double  B0;
397
398             //CvPoint Center = cvPoint(cvRound(m_Blob.x),cvRound(m_Blob.y));
399             CollectHist(pImg, NULL, &m_Blob, &m_HistCandidate);
400             B0 = calcBhattacharyya();
401
402             if(m_Wnd)if(CV_BLOB_ID(pBlobPrev)==0 && iter == 0)
403             {   /* Debug: */
404                 IplImage*   pW = cvCloneImage(pImgFG);
405                 IplImage*   pWFG = cvCloneImage(pImgFG);
406                 int         x,y;
407
408                 cvZero(pW);
409                 cvZero(pWFG);
410
411                 assert(m_ObjSize.width < pImg->width);
412                 assert(m_ObjSize.height < pImg->height);
413
414                 /* Calculate shift vector: */
415                 for(y=0; y<pImg->height; ++y)
416                 {
417                     unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y,0);
418                     unsigned char* pMaskData = pImgFG?(&CV_IMAGE_ELEM(pImgFG,unsigned char,y,0)):NULL;
419
420                     for(x=0; x<pImg->width; ++x, pImgData+=3)
421                     {
422                         int         xk = cvRound(x-(m_Blob.x-m_Blob.w*0.5));
423                         int         yk = cvRound(y-(m_Blob.y-m_Blob.h*0.5));
424                         double      HM = 0;
425                         double      HC = 0;
426                         double      K;
427                         int index = HIST_INDEX(pImgData);
428                         assert(index >= 0 && index < m_BinNumTotal);
429
430                         if(fabs(x-m_Blob.x)>m_Blob.w*0.6) continue;
431                         if(fabs(y-m_Blob.y)>m_Blob.h*0.6) continue;
432
433                         if(xk < 0 || xk >= m_KernelMeanShift->cols) continue;
434                         if(yk < 0 || yk >= m_KernelMeanShift->rows) continue;
435
436                         if(m_HistModel.m_HistVolume>0)
437                             HM = ((DefHistType*)m_HistModel.m_pHist->data.ptr)[index]/m_HistModel.m_HistVolume;
438
439                         if(m_HistCandidate.m_HistVolume>0)
440                             HC = ((DefHistType*)m_HistCandidate.m_pHist->data.ptr)[index]/m_HistCandidate.m_HistVolume;
441
442                         K = *(DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelMeanShift[0],yk,xk,sizeof(DefHistType));
443
444                         if(HC>0)
445                         {
446                             double  V = sqrt(HM / HC);
447                             int     Vi = cvRound(V * 64);
448                             if(Vi < 0) Vi = 0;
449                             if(Vi > 255) Vi = 255;
450                             CV_IMAGE_ELEM(pW,uchar,y,x) = (uchar)Vi;
451
452                             V += m_FGWeight*(pMaskData?(pMaskData[x]/255.0f):0);
453                             V*=K;
454                             Vi = cvRound(V * 64);
455                             if(Vi < 0) Vi = 0;
456                             if(Vi > 255) Vi = 255;
457                             CV_IMAGE_ELEM(pWFG,uchar,y,x) = (uchar)Vi;
458                         }
459
460                     }   /* Next column. */
461                 }   /*  Next row. */
462
463                 //cvNamedWindow("MSFG_W",0);
464                 //cvShowImage("MSFG_W",pW);
465                 //cvNamedWindow("MSFG_WFG",0);
466                 //cvShowImage("MSFG_WFG",pWFG);
467                 //cvNamedWindow("MSFG_FG",0);
468                 //cvShowImage("MSFG_FG",pImgFG);
469
470                 //cvSaveImage("MSFG_W.bmp",pW);
471                 //cvSaveImage("MSFG_WFG.bmp",pWFG);
472                 //cvSaveImage("MSFG_FG.bmp",pImgFG);
473
474             }   /* Debug. */
475
476
477             /* Calculate new position by meanshift: */
478
479             /* Calculate new position: */
480             if(m_Dim == 3)
481             {
482                 int         x0 = cvRound(m_Blob.x - m_ObjSize.width*0.5);
483                 int         y0 = cvRound(m_Blob.y - m_ObjSize.height*0.5);
484                 int         x,y;
485
486                 assert(m_ObjSize.width < pImg->width);
487                 assert(m_ObjSize.height < pImg->height);
488
489                 /* Crop blob bounds: */
490                 if((x0+m_ObjSize.width)>=pImg->width) x0=pImg->width-m_ObjSize.width-1;
491                 if((y0+m_ObjSize.height)>=pImg->height) y0=pImg->height-m_ObjSize.height-1;
492                 if(x0<0){ x0=0;}
493                 if(y0<0){ y0=0;}
494
495                 /* Calculate shift vector: */
496                 for(y=0; y<m_ObjSize.height; ++y)
497                 {
498                     unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y+y0,x0*3);
499                     unsigned char* pMaskData = pImgFG?(&CV_IMAGE_ELEM(pImgFG,unsigned char,y+y0,x0)):NULL;
500                     DefHistType* pKernelData = (DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelMeanShift[0],y,0,sizeof(DefHistType));
501
502                     for(x=0; x<m_ObjSize.width; ++x, pImgData+=3)
503                     {
504                         DefHistType K = pKernelData[x];
505                         double      HM = 0;
506                         double      HC = 0;
507                         int index = HIST_INDEX(pImgData);
508                         assert(index >= 0 && index < m_BinNumTotal);
509
510                         if(m_HistModel.m_HistVolume>0)
511                             HM = ((DefHistType*)m_HistModel.m_pHist->data.ptr)[index]/m_HistModel.m_HistVolume;
512
513                         if(m_HistCandidate.m_HistVolume>0)
514                             HC = ((DefHistType*)m_HistCandidate.m_pHist->data.ptr)[index]/m_HistCandidate.m_HistVolume;
515
516                         if(HC>0)
517                         {
518                             double V = sqrt(HM / HC);
519                             if(!m_Collision && m_FGWeight>0 && pMaskData)
520                             {
521                                 V += m_FGWeight*(pMaskData[x]/255.0f);
522                             }
523                             K *= (float)MIN(V,100000.);
524                         }
525
526                         sum += K;
527                         newx += K*x;
528                         newy += K*y;
529                     }   /* Next column. */
530                 }   /*  Next row. */
531
532                 if(sum > 0)
533                 {
534                     newx /= sum;
535                     newy /= sum;
536                 }
537                 newx += x0;
538                 newy += y0;
539
540             }   /* if m_Dim == 3. */
541
542             /* Calculate new position by meanshift: */
543
544             for(;;)
545             {   /* Iterate using bahattcharrya coefficient: */
546                 double  B1;
547                 CvBlob  B = m_Blob;
548 //                CvPoint NewCenter = cvPoint(cvRound(newx),cvRound(newy));
549                 B.x = newx;
550                 B.y = newy;
551                 CollectHist(pImg, NULL, &B, &m_HistCandidate);
552                 B1 = calcBhattacharyya();
553                 if(B1 > B0) break;
554                 newx = 0.5f*(newx+m_Blob.x);
555                 newy = 0.5f*(newy+m_Blob.y);
556                 if(fabs(newx-m_Blob.x)<0.1 && fabs(newy-m_Blob.y)<0.1) break;
557             }   /* Iterate using bahattcharrya coefficient. */
558
559             if(fabs(newx-m_Blob.x)<0.5 && fabs(newy-m_Blob.y)<0.5) break;
560             m_Blob.x = newx;
561             m_Blob.y = newy;
562         }   /* Next iteration. */
563
564         while(!m_Collision && m_FGWeight>0)
565         {   /* Update size if no collision. */
566             float       Alpha = 0.04f;
567             CvBlob      NewBlob;
568             double      M00,X,Y,XX,YY;
569             CvMoments   m;
570             CvRect      r;
571             CvMat       mat;
572
573             r.width = cvRound(m_Blob.w*1.5+0.5);
574             r.height = cvRound(m_Blob.h*1.5+0.5);
575             r.x = cvRound(m_Blob.x - 0.5*r.width);
576             r.y = cvRound(m_Blob.y - 0.5*r.height);
577             if(r.x < 0) break;
578             if(r.y < 0) break;
579             if(r.x+r.width >= pImgFG->width) break;
580             if(r.y+r.height >= pImgFG->height) break;
581             if(r.height < 5 || r.width < 5) break;
582
583             cvMoments( cvGetSubRect(pImgFG,&mat,r), &m, 0 );
584             M00 = cvGetSpatialMoment( &m, 0, 0 );
585             if(M00 <= 0 ) break;
586             X = cvGetSpatialMoment( &m, 1, 0 )/M00;
587             Y = cvGetSpatialMoment( &m, 0, 1 )/M00;
588             XX = (cvGetSpatialMoment( &m, 2, 0 )/M00) - X*X;
589             YY = (cvGetSpatialMoment( &m, 0, 2 )/M00) - Y*Y;
590             NewBlob = cvBlob(r.x+(float)X,r.y+(float)Y,(float)(4*sqrt(XX)),(float)(4*sqrt(YY)));
591
592             NewBlob.w = Alpha*NewBlob.w+m_Blob.w*(1-Alpha);
593             NewBlob.h = Alpha*NewBlob.h+m_Blob.h*(1-Alpha);
594
595             m_Blob.w = MAX(NewBlob.w,5);
596             m_Blob.h = MAX(NewBlob.h,5);
597             break;
598
599         }   /* Update size if no collision. */
600
601         return &m_Blob;
602
603     };  /* CvBlobTrackerOneMSFG::Process */
604
605     virtual double GetConfidence(CvBlob* pBlob, IplImage* pImg, IplImage* /*pImgFG*/ = NULL, IplImage* pImgUnusedReg = NULL)
606     {
607         double  S = 0.2;
608         double  B = GetBhattacharyya(pImg, pImgUnusedReg, pBlob, &m_HistTemp);
609         return exp((B-1)/(2*S));
610
611     };  /*CvBlobTrackerOneMSFG::*/
612
613     virtual void Update(CvBlob* pBlob, IplImage* pImg, IplImage* pImgFG = NULL)
614     {   /* Update histogram: */
615         UpdateModelHist(pImg, pImgFG, pBlob?pBlob:&m_Blob);
616     }   /*CvBlobTrackerOneMSFG::*/
617
618     virtual void Release(){delete this;};
619     virtual void SetCollision(int CollisionFlag)
620     {
621         m_Collision = CollisionFlag;
622     }
623     virtual void SaveState(CvFileStorage* fs)
624     {
625         cvWriteStruct(fs, "Blob", &m_Blob, "ffffi");
626         cvWriteInt(fs,"Collision", m_Collision);
627         cvWriteInt(fs,"HistVolume", cvRound(m_HistModel.m_HistVolume));
628         cvWrite(fs,"Hist", m_HistModel.m_pHist);
629     };
630     virtual void LoadState(CvFileStorage* fs, CvFileNode* node)
631     {
632         CvMat* pM;
633         cvReadStructByName(fs, node, "Blob",&m_Blob, "ffffi");
634         m_Collision = cvReadIntByName(fs,node,"Collision",m_Collision);
635         pM = (CvMat*)cvRead(fs,cvGetFileNodeByName(fs,node,"Hist"));
636         if(pM)
637         {
638             m_HistModel.m_pHist = pM;
639             m_HistModel.m_HistVolume = (float)cvSum(pM).val[0];
640         }
641     };
642
643 };  /*CvBlobTrackerOneMSFG*/
644
645 #if 0
646 void CvBlobTrackerOneMSFG::CollectHist(IplImage* pImg, IplImage* pMask, CvBlob* pBlob, DefHist* pHist)
647 {
648     int UsePrecalculatedKernel = 0;
649     int BW = cvRound(pBlob->w);
650     int BH = cvRound(pBlob->h);
651     DefHistType Volume = 0;
652     int         x0 = cvRound(pBlob->x - BW*0.5);
653     int         y0 = cvRound(pBlob->y - BH*0.5);
654     int         x,y;
655
656     UsePrecalculatedKernel = (BW == m_ObjSize.width && BH == m_ObjSize.height ) ;
657
658     //cvZero(pHist);
659     cvSet(pHist->m_pHist,cvScalar(1.0/m_BinNumTotal)); /* no zero bins, all bins have very small value*/
660     Volume = 1;
661
662     assert(BW < pImg->width);
663     assert(BH < pImg->height);
664     if((x0+BW)>=pImg->width) BW=pImg->width-x0-1;
665     if((y0+BH)>=pImg->height) BH=pImg->height-y0-1;
666     if(x0<0){ x0=0;}
667     if(y0<0){ y0=0;}
668
669     if(m_Dim == 3)
670     {
671         for(y=0; y<BH; ++y)
672         {
673             unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y+y0,x0*3);
674             unsigned char* pMaskData = pMask?(&CV_IMAGE_ELEM(pMask,unsigned char,y+y0,x0)):NULL;
675             DefHistType* pKernelData = NULL;
676
677             if(UsePrecalculatedKernel)
678             {
679                 pKernelData = ((DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelHist[0],y,0,sizeof(DefHistType)));
680             }
681
682             for(x=0; x<BW; ++x, pImgData+=3)
683             {
684                 DefHistType K;
685                 int index = HIST_INDEX(pImgData);
686                 assert(index >= 0 && index < pHist->m_pHist->cols);
687
688                 if(UsePrecalculatedKernel)
689                 {
690                     K = pKernelData[x];
691                 }
692                 else
693                 {
694                     float dx = (x+x0-pBlob->x)/(pBlob->w*0.5);
695                     float dy = (y+y0-pBlob->y)/(pBlob->h*0.5);
696                     double r2 = dx*dx+dy*dy;
697                     K = GetKernelHist(r2);
698                 }
699
700                 if(pMaskData)
701                 {
702                     K *= pMaskData[x]*0.003921568627450980392156862745098;
703                 }
704                 Volume += K;
705                 ((DefHistType*)(pHist->m_pHist->data.ptr))[index] += K;
706
707             }   /* Next column. */
708         }   /*  Next row. */
709     }   /*  if m_Dim == 3. */
710
711     pHist->m_HistVolume = Volume;
712
713 };  /* CollectHist */
714 #endif
715
716 CvBlobTrackerOne* cvCreateBlobTrackerOneMSFG()
717 {
718     return (CvBlobTrackerOne*) new CvBlobTrackerOneMSFG;
719 }
720
721 CvBlobTracker* cvCreateBlobTrackerMSFG()
722 {
723     return cvCreateBlobTrackerList(cvCreateBlobTrackerOneMSFG);
724 }
725
726 /* Create specific tracker without FG
727  * usin - just simple mean-shift tracker: */
728 class CvBlobTrackerOneMS:public CvBlobTrackerOneMSFG
729 {
730 public:
731     CvBlobTrackerOneMS()
732     {
733         SetParam("FGWeight",0);
734         DelParam("FGWeight");
735     };
736 };
737
738 CvBlobTrackerOne* cvCreateBlobTrackerOneMS()
739 {
740     return (CvBlobTrackerOne*) new CvBlobTrackerOneMS;
741 }
742
743 CvBlobTracker* cvCreateBlobTrackerMS()
744 {
745     return cvCreateBlobTrackerList(cvCreateBlobTrackerOneMS);
746 }
747
748 typedef struct DefParticle
749 {
750     CvBlob  blob;
751     float   Vx,Vy;
752     double  W;
753 } DefParticle;
754
755 class CvBlobTrackerOneMSPF:public CvBlobTrackerOneMS
756 {
757 private:
758     /* parameters */
759     int             m_ParticleNum;
760     float           m_UseVel;
761     float           m_SizeVar;
762     float           m_PosVar;
763
764     CvSize          m_ImgSize;
765     CvBlob          m_Blob;
766     DefParticle*    m_pParticlesPredicted;
767     DefParticle*    m_pParticlesResampled;
768     CvRNG           m_RNG;
769 #ifdef _OPENMP
770     int             m_ThreadNum;
771     DefHist*        m_HistForParalel;
772 #endif
773
774 public:
775     virtual void SaveState(CvFileStorage* fs)
776     {
777         CvBlobTrackerOneMS::SaveState(fs);
778         cvWriteInt(fs,"ParticleNum",m_ParticleNum);
779         cvWriteStruct(fs,"ParticlesPredicted",m_pParticlesPredicted,"ffffiffd",m_ParticleNum);
780         cvWriteStruct(fs,"ParticlesResampled",m_pParticlesResampled,"ffffiffd",m_ParticleNum);
781     };
782
783     virtual void LoadState(CvFileStorage* fs, CvFileNode* node)
784     {
785         //CvMat* pM;
786         CvBlobTrackerOneMS::LoadState(fs,node);
787         m_ParticleNum = cvReadIntByName(fs,node,"ParticleNum",m_ParticleNum);
788         if(m_ParticleNum>0)
789         {
790             Realloc();
791             printf("sizeof(DefParticle) is %d\n", sizeof(DefParticle));
792             cvReadStructByName(fs,node,"ParticlesPredicted",m_pParticlesPredicted,"ffffiffd");
793             cvReadStructByName(fs,node,"ParticlesResampled",m_pParticlesResampled,"ffffiffd");
794         }
795     };
796     CvBlobTrackerOneMSPF()
797     {
798         m_pParticlesPredicted = NULL;
799         m_pParticlesResampled = NULL;
800         m_ParticleNum = 200;
801
802         AddParam("ParticleNum",&m_ParticleNum);
803         CommentParam("ParticleNum","Number of particles");
804         Realloc();
805
806         m_UseVel = 0;
807         AddParam("UseVel",&m_UseVel);
808         CommentParam("UseVel","Percent of particles which use velocity feature");
809
810         m_SizeVar = 0.05f;
811         AddParam("SizeVar",&m_SizeVar);
812         CommentParam("SizeVar","Size variation (in object size)");
813
814         m_PosVar = 0.2f;
815         AddParam("PosVar",&m_PosVar);
816         CommentParam("PosVar","Position variation (in object size)");
817
818         m_RNG = cvRNG(0);
819 #ifdef _OPENMP
820         {
821             m_ThreadNum = omp_get_num_procs();
822             m_HistForParalel = new DefHist[m_ThreadNum];
823         }
824 #endif
825     }
826
827     ~CvBlobTrackerOneMSPF()
828     {
829         if(m_pParticlesResampled)cvFree(&m_pParticlesResampled);
830         if(m_pParticlesPredicted)cvFree(&m_pParticlesPredicted);
831 #ifdef _OPENMP
832         if(m_HistForParalel) delete[] m_HistForParalel;
833 #endif
834     }
835
836 private:
837     void Realloc()
838     {
839         if(m_pParticlesResampled)cvFree(&m_pParticlesResampled);
840         if(m_pParticlesPredicted)cvFree(&m_pParticlesPredicted);
841         m_pParticlesPredicted = (DefParticle*)cvAlloc(sizeof(DefParticle)*m_ParticleNum);
842         m_pParticlesResampled = (DefParticle*)cvAlloc(sizeof(DefParticle)*m_ParticleNum);
843     };  /* Realloc*/
844
845     void DrawDebug(IplImage* pImg, IplImage* /*pImgFG*/)
846     {
847         int k;
848         for(k=0; k<2; ++k)
849         {
850             DefParticle*    pBP = k?m_pParticlesResampled:m_pParticlesPredicted;
851             //const char*   name = k?"MSPF resampled particle":"MSPF Predicted particle";
852             IplImage*       pI = cvCloneImage(pImg);
853             int             h,hN = m_ParticleNum;
854             CvBlob          C = cvBlob(0,0,0,0);
855             double          WS = 0;
856             for(h=0; h<hN; ++h)
857             {
858                 CvBlob  B = pBP[h].blob;
859                 int     CW = cvRound(255*pBP[h].W);
860                 CvBlob* pB = &B;
861                 int x = cvRound(CV_BLOB_RX(pB)), y = cvRound(CV_BLOB_RY(pB));
862                 CvSize  s = cvSize(MAX(1,x), MAX(1,y));
863                 double  W = pBP[h].W;
864                 C.x += pB->x;
865                 C.y += pB->y;
866                 C.w += pB->w;
867                 C.h += pB->h;
868                 WS+=W;
869
870                 s = cvSize(1,1);
871                 cvEllipse( pI,
872                     cvPointFrom32f(CV_BLOB_CENTER(pB)),
873                     s,
874                     0, 0, 360,
875                     CV_RGB(CW,0,0), 1 );
876
877             }   /* Next hypothesis. */
878
879             C.x /= hN;
880             C.y /= hN;
881             C.w /= hN;
882             C.h /= hN;
883
884             cvEllipse( pI,
885                 cvPointFrom32f(CV_BLOB_CENTER(&C)),
886                 cvSize(cvRound(C.w*0.5),cvRound(C.h*0.5)),
887                 0, 0, 360,
888                 CV_RGB(0,0,255), 1 );
889
890             cvEllipse( pI,
891                 cvPointFrom32f(CV_BLOB_CENTER(&m_Blob)),
892                 cvSize(cvRound(m_Blob.w*0.5),cvRound(m_Blob.h*0.5)),
893                 0, 0, 360,
894                 CV_RGB(0,255,0), 1 );
895
896             //cvNamedWindow(name,0);
897             //cvShowImage(name,pI);
898             cvReleaseImage(&pI);
899         } /*  */
900
901         //printf("Blob %d, point (%.1f,%.1f) size (%.1f,%.1f)\n",m_Blob.ID,m_Blob.x,m_Blob.y,m_Blob.w,m_Blob.h);
902     } /* ::DrawDebug */
903
904 private:
905     void Prediction()
906     {
907         int p;
908         for(p=0; p<m_ParticleNum; ++p)
909         {   /* "Prediction" of particle: */
910             //double  t;
911             float   r[5];
912             CvMat   rm = cvMat(1,5,CV_32F,r);
913             cvRandArr(&m_RNG,&rm,CV_RAND_NORMAL,cvScalar(0),cvScalar(1));
914
915             m_pParticlesPredicted[p] = m_pParticlesResampled[p];
916
917             if(cvRandReal(&m_RNG)<0.5)
918             {   /* Half of particles will predict based on external blob: */
919                 m_pParticlesPredicted[p].blob = m_Blob;
920             }
921
922             if(cvRandReal(&m_RNG)<m_UseVel)
923             {   /* Predict moving particle by usual way by using speed: */
924                 m_pParticlesPredicted[p].blob.x += m_pParticlesPredicted[p].Vx;
925                 m_pParticlesPredicted[p].blob.y += m_pParticlesPredicted[p].Vy;
926             }
927             else
928             {   /* Stop several particles: */
929                 m_pParticlesPredicted[p].Vx = 0;
930                 m_pParticlesPredicted[p].Vy = 0;
931             }
932
933             {   /* Update position: */
934                 float S = (m_Blob.w + m_Blob.h)*0.5f;
935                 m_pParticlesPredicted[p].blob.x += m_PosVar*S*r[0];
936                 m_pParticlesPredicted[p].blob.y += m_PosVar*S*r[1];
937
938                 /* Update velocity: */
939                 m_pParticlesPredicted[p].Vx += (float)(m_PosVar*S*0.1*r[3]);
940                 m_pParticlesPredicted[p].Vy += (float)(m_PosVar*S*0.1*r[4]);
941             }
942
943             /* Update size: */
944             m_pParticlesPredicted[p].blob.w *= (1+m_SizeVar*r[2]);
945             m_pParticlesPredicted[p].blob.h *= (1+m_SizeVar*r[2]);
946
947             /* Truncate size of particle: */
948             if(m_pParticlesPredicted[p].blob.w > m_ImgSize.width*0.5f)
949             {
950                 m_pParticlesPredicted[p].blob.w = m_ImgSize.width*0.5f;
951             }
952
953             if(m_pParticlesPredicted[p].blob.h > m_ImgSize.height*0.5f)
954             {
955                 m_pParticlesPredicted[p].blob.h = m_ImgSize.height*0.5f;
956             }
957
958             if(m_pParticlesPredicted[p].blob.w < 1 )
959             {
960                 m_pParticlesPredicted[p].blob.w = 1;
961             }
962
963             if(m_pParticlesPredicted[p].blob.h < 1)
964             {
965                 m_pParticlesPredicted[p].blob.h = 1;
966             }
967         }   /* "Prediction" of particle. */
968     }   /* Prediction */
969
970     void UpdateWeightsMS(IplImage* pImg, IplImage* /*pImgFG*/)
971     {
972         int p;
973 #ifdef _OPENMP
974         if( m_HistForParalel[0].m_pHist==NULL || m_HistForParalel[0].m_pHist->cols != m_BinNumTotal)
975         {
976             int t;
977             for(t=0; t<m_ThreadNum; ++t)
978                 m_HistForParalel[t].Resize(m_BinNumTotal);
979         }
980 #endif
981
982 #ifdef _OPENMP
983 #pragma omp parallel for num_threads(m_ThreadNum),schedule(runtime)
984 #endif
985         for(p=0;p<m_ParticleNum;++p)
986         {   /* Calculate weights for particles: */
987             double  S = 0.2;
988             double  B = 0;
989 #ifdef _OPENMP
990             assert(omp_get_thread_num()<m_ThreadNum);
991 #endif
992
993             B = GetBhattacharyya(
994                 pImg, NULL,
995                 &(m_pParticlesPredicted[p].blob)
996 #ifdef _OPENMP
997                 ,&(m_HistForParalel[omp_get_thread_num()])
998 #endif
999                 );
1000             m_pParticlesPredicted[p].W *= exp((B-1)/(2*S));
1001
1002         }   /* Calculate weights for particles. */
1003     }
1004
1005     void UpdateWeightsCC(IplImage* /*pImg*/, IplImage* /*pImgFG*/)
1006     {
1007         int p;
1008 #ifdef _OPENMP
1009 #pragma omp parallel for
1010 #endif
1011         for(p=0; p<m_ParticleNum; ++p)
1012         {   /* Calculate weights for particles: */
1013             double W = 1;
1014             m_pParticlesPredicted[p].W *= W;
1015         }   /* Calculate weights for particles. */
1016     }
1017
1018     void Resample()
1019     {   /* Resample particle: */
1020         int         p;
1021         double      Sum = 0;
1022
1023         for(p=0; p<m_ParticleNum; ++p)
1024         {
1025             Sum += m_pParticlesPredicted[p].W;
1026         }
1027
1028         for(p=0; p<m_ParticleNum; ++p)
1029         {
1030             double  T = Sum * cvRandReal(&m_RNG);   /* Set current random threshold for cululative weight. */
1031             int     p2;
1032             double  Sum2 = 0;
1033
1034             for(p2=0; p2<m_ParticleNum; ++p2)
1035             {
1036                 Sum2 += m_pParticlesPredicted[p2].W;
1037                 if(Sum2 >= T)break;
1038             }
1039
1040             if(p2>=m_ParticleNum)p2=m_ParticleNum-1;
1041             m_pParticlesResampled[p] = m_pParticlesPredicted[p2];
1042             m_pParticlesResampled[p].W = 1;
1043
1044         }   /* Find next particle. */
1045     }   /*  Resample particle. */
1046
1047
1048 public:
1049     virtual void Init(CvBlob* pBlobInit, IplImage* pImg, IplImage* pImgFG = NULL)
1050     {
1051         int i;
1052         CvBlobTrackerOneMSFG::Init(pBlobInit, pImg, pImgFG);
1053         DefParticle PP;
1054         PP.W = 1;
1055         PP.Vx = 0;
1056         PP.Vy = 0;
1057         PP.blob = pBlobInit[0];
1058         for(i=0;i<m_ParticleNum;++i)
1059         {
1060             m_pParticlesPredicted[i] = PP;
1061             m_pParticlesResampled[i] = PP;
1062         }
1063         m_Blob = pBlobInit[0];
1064
1065     }   /* CvBlobTrackerOneMSPF::Init*/
1066
1067     virtual CvBlob* Process(CvBlob* pBlobPrev, IplImage* pImg, IplImage* pImgFG = NULL)
1068     {
1069         int p;
1070
1071         m_ImgSize.width = pImg->width;
1072         m_ImgSize.height = pImg->height;
1073
1074
1075         m_Blob = pBlobPrev[0];
1076
1077         {   /* Check blob size and realloc kernels if it is necessary: */
1078             int w = cvRound(m_Blob.w);
1079             int h = cvRound(m_Blob.h);
1080             if( w != m_ObjSize.width || h!=m_ObjSize.height)
1081             {
1082                 ReAllocKernel(w,h);
1083                 /* After this ( w != m_ObjSize.width || h!=m_ObjSize.height) should be false. */
1084             }
1085         }   /* Check blob size and realloc kernels if it is necessary. */
1086
1087         Prediction();
1088
1089 #ifdef REPORT_TICKS
1090         int64 ticks = cvGetTickCount();
1091 #endif
1092
1093         UpdateWeightsMS(pImg, pImgFG);
1094
1095 #ifdef REPORT_TICKS
1096         ticks = cvGetTickCount() - ticks;
1097         fprintf(stderr, "PF UpdateWeights, %d ticks\n",  (int)ticks);
1098         ticks = cvGetTickCount();
1099 #endif
1100
1101         Resample();
1102
1103 #ifdef REPORT_TICKS
1104         ticks = cvGetTickCount() - ticks;
1105         fprintf(stderr, "PF Resampling, %d ticks\n",  (int)ticks);
1106 #endif
1107
1108         {   /* Find average result: */
1109             float   x = 0;
1110             float   y = 0;
1111             float   w = 0;
1112             float   h = 0;
1113             float   Sum = 0;
1114
1115             DefParticle* pP = m_pParticlesResampled;
1116
1117             for(p=0; p<m_ParticleNum; ++p)
1118             {
1119                 float W = (float)pP[p].W;
1120                 x += W*pP[p].blob.x;
1121                 y += W*pP[p].blob.y;
1122                 w += W*pP[p].blob.w;
1123                 h += W*pP[p].blob.h;
1124                 Sum += W;
1125             }
1126
1127             if(Sum>0)
1128             {
1129                 m_Blob.x = x / Sum;
1130                 m_Blob.y = y / Sum;
1131                 m_Blob.w = w / Sum;
1132                 m_Blob.h = h / Sum;
1133             }
1134         }   /* Find average result. */
1135
1136         if(m_Wnd)
1137         {
1138             DrawDebug(pImg, pImgFG);
1139         }
1140
1141         return &m_Blob;
1142
1143     }   /* CvBlobTrackerOneMSPF::Process */
1144
1145     virtual void SkipProcess(CvBlob* pBlob, IplImage* /*pImg*/, IplImage* /*pImgFG*/ = NULL)
1146     {
1147         int p;
1148         for(p=0; p<m_ParticleNum; ++p)
1149         {
1150             m_pParticlesResampled[p].blob = pBlob[0];
1151             m_pParticlesResampled[p].Vx = 0;
1152             m_pParticlesResampled[p].Vy = 0;
1153             m_pParticlesResampled[p].W = 1;
1154         }
1155     }
1156
1157     virtual void Release(){delete this;};
1158     virtual void ParamUpdate()
1159     {
1160         Realloc();
1161     }
1162
1163 };  /* CvBlobTrackerOneMSPF */
1164
1165 CvBlobTrackerOne* cvCreateBlobTrackerOneMSPF()
1166 {
1167     return (CvBlobTrackerOne*) new CvBlobTrackerOneMSPF;
1168 }
1169
1170 CvBlobTracker* cvCreateBlobTrackerMSPF()
1171 {
1172     return cvCreateBlobTrackerList(cvCreateBlobTrackerOneMSPF);
1173 }
1174