1 /*M///////////////////////////////////////////////////////////////////////////////////////
3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
5 // By downloading, copying, installing or using the software you agree to this license.
6 // If you do not agree to this license, do not download, install,
7 // copy or use the software.
10 // Intel License Agreement
11 // For Open Source Computer Vision Library
13 // Copyright (C) 2000, Intel Corporation, all rights reserved.
14 // Third party copyrights are property of their respective owners.
16 // Redistribution and use in source and binary forms, with or without modification,
17 // are permitted provided that the following conditions are met:
19 // * Redistribution's of source code must retain the above copyright notice,
20 // this list of conditions and the following disclaimer.
22 // * Redistribution's in binary form must reproduce the above copyright notice,
23 // this list of conditions and the following disclaimer in the documentation
24 // and/or other materials provided with the distribution.
26 // * The name of Intel Corporation may not be used to endorse or promote products
27 // derived from this software without specific prior written permission.
29 // This software is provided by the copyright holders and contributors "as is" and
30 // any express or implied warranties, including, but not limited to, the implied
31 // warranties of merchantability and fitness for a particular purpose are disclaimed.
32 // In no event shall the Intel Corporation or contributors be liable for any direct,
33 // indirect, incidental, special, exemplary, or consequential damages
34 // (including, but not limited to, procurement of substitute goods or services;
35 // loss of use, data, or profits; or business interruption) however caused
36 // and on any theory of liability, whether in contract, strict liability,
37 // or tort (including negligence or otherwise) arising in any way out of
38 // the use of this software, even if advised of the possibility of such damage.
40 //M*/// DynGestServer.cpp: implementation of the CDynGestServer class.
42 //////////////////////////////////////////////////////////////////////
45 #include "DynGestServer.h"
56 CDynGesture::CDynGesture( int num_states, int num_mix )
60 m_obsType = OT_VELOCITY;
62 m_obsType = OT_VELOCITY_ORIENTATION;
65 m_num_states = num_states;
70 CDynGesture::~CDynGesture()
72 if ( m_hmm ) icvRelease1DHMM( &m_hmm );
74 for ( int i = 0; i < m_Sequences.size() ; i++ )
76 cvReleaseObsInfo( &(m_Sequences[i]) );
80 void CDynGesture::AddSequence( float* vectors, int seq_size, int vect_size )
82 Cv1DObsInfo* obs_info;
83 obs_info = cvCreateObsInfo( cvSize( seq_size,1), vect_size );
85 memcpy( obs_info->obs, vectors, seq_size * vect_size * sizeof(float) );
86 m_Sequences.push_back( obs_info );
92 void CDynGesture::Train()
94 ExtractObservations();
96 int num_seq = m_Sequences.size();
98 if ( !num_seq ) ASSERT(0);
101 if (!m_hmm) CreateHMM();
103 for(int i = 0; i < num_seq; i++ )
105 icvUniform1DSegm( m_Sequences[i], m_hmm );
106 //printf("uniform segmentation performed\n");
109 icvInit1DMixSegm( &m_Sequences[0], num_seq, m_hmm );
110 //printf("Init 1D mix segmentation performed\n");
113 float old_likelihood = 0;
115 //printf("Begin training\n");
120 icvEstimate1DHMMStateParams( &m_Sequences[0], num_seq, m_hmm );
121 //printf("HMMstate performed\n");
122 icvEstimate1DTransProb( &m_Sequences[0], num_seq, m_hmm );
123 //printf("Transprob performed\n");
124 float likelihood = 0;
126 for( int j = 0; j < num_seq; j++ )
128 icvEstimate1DObsProb( m_Sequences[j], m_hmm );
129 likelihood += icvViterbi( m_Sequences[j], m_hmm );
132 //printf("likelihood %f\n", likelihood );
134 icv1DMixSegmL2( &m_Sequences[0], num_seq, m_hmm );
135 //printf("1DMixSegm performed\n");
137 trained = ( fabs(likelihood - old_likelihood) < 0.01 );
138 old_likelihood = likelihood;
140 //printf("%d hmm trained\n", h );
145 BOOL CDynGesture::SaveHMM(const char* filename)
149 if (!m_hmm) return FALSE;
150 file = fopen( filename, "wt" );
151 if (!file) { assert(0); return false; }
154 fprintf(file, "%s %d\n", "<NumStates>", m_hmm->num_states );
155 fprintf(file, "%s ", "<NumMixtures>");
156 for( int i = 0; i < m_hmm->num_states; i++ )
158 fprintf(file, "%d ", m_hmm->u.state[i].num_mix );
162 fprintf(file, "%s %d\n", "<VecSize>", m_vect_size );
164 //consequently write all hmms
166 if (hmm->level == 0 )
168 for ( int j = 0; j < hmm->num_states; j++ )
170 CvEHMMState* state = &(hmm->u.state[j]);
172 fprintf(file, "%s %d\n", "<State>", j);
173 fprintf(file, "%s %d\n", "<NumMixes>", state->num_mix);
175 float* mu = state->mu;
176 float* inv_var = state->inv_var;
178 for( int m = 0; m < state->num_mix; m++)
180 fprintf(file, "%s %d %s %f\n", "<Mixture>", m, "<Weight>", state->weight[m] );
181 fprintf(file, "%s\n", "<Mean>");
183 for (int k = 0; k < m_vect_size; k++)
185 fprintf(file, "%f ", mu[0]);
190 fprintf(file, "%s\n", "<Inverted_Deviation>");
192 for (k = 0; k < m_vect_size; k++)
194 fprintf(file, "%f ", inv_var[0]);
199 fprintf(file, "%s %f\n", "<LogVarVal>", state->log_var_val[m] );
204 //write the transition probability matrix
205 fprintf(file, "%s\n", "<TransP>" );
206 float* prob = hmm->transP;
208 for (int j = 0; j < hmm->num_states; j++)
210 for (int k = 0; k < hmm->num_states; k++)
212 fprintf(file, "%f ", *prob);
222 BOOL CDynGesture::LoadHMM(const char* filename)
230 if (m_hmm) icvRelease1DHMM( &m_hmm);
232 file = fopen( filename, "rt" );
234 if (!file) return false;
237 fscanf(file, "%s %d\n", temp_char, &num_states);
238 fscanf(file, "%s ", temp_char);
240 //compute total number of internal states
241 int total_states = num_states;
243 //read number of mixtures
244 //fscanf(file, "%s ", temp_char);
245 for( int i = 0; i < total_states; i++ )
247 fscanf(file, "%d ", &num_mix[i] );
251 fscanf(file, "%s %d\n", temp_char, &vec_size);
254 icvCreate1DHMM( &m_hmm, num_states, num_mix, vec_size);
256 //create HMM with known parameters
257 //!!! cvCreate2DHMM( &m_hmm, num_states, num_mix, m_vectSize);
258 if (!m_hmm ) return FALSE;
260 //consequently read all hmms
264 for (int j = 0; j < num_states; j++)
266 CvEHMMState* state = &(hmm->u.state[j]);
268 fscanf(file, "%s %d\n", temp_char, &temp_int); assert(temp_int == j);
270 fscanf(file, "%s %d\n", temp_char, &temp_int); assert(temp_int == state->num_mix);
272 float* mu = state->mu;
273 float* inv_var = state->inv_var;
275 for( int m = 0; m < state->num_mix; m++)
278 fscanf(file, "%s %d %s %f\n", temp_char, &temp_int, temp_char, &(state->weight[m]) );
279 assert( temp_int == m );
280 fscanf(file, "%s\n", temp_char );
282 for (int k = 0; k < vec_size; k++)
284 fscanf(file, "%f ", mu);
289 fscanf(file, "%s\n", temp_char); // <Inverted_Deviation>
291 for (k = 0; k < vec_size; k++)
293 fscanf(file, "%f ", inv_var);
298 fscanf(file, "%s %f\n", temp_char, &(state->log_var_val[m]) ); // <LogVarVal>
302 //read the transition probability matrix
303 fscanf(file, "%s\n", temp_char ); // <TransP>
304 float* prob = hmm->transP;
306 for ( j = 0; j < hmm->num_states; j++)
308 for (int k = 0; k < hmm->num_states; k++)
310 fscanf(file, "%f ", prob);
321 int CDynGesture::GetSeqNumber()
323 return m_Params.size();
326 //////////////////////////////////////////////////////////////////////
327 // Construction/Destruction
328 //////////////////////////////////////////////////////////////////////
330 CDynGestServer::CDynGestServer()
332 m_fullTrained = FALSE;
335 CDynGestServer::~CDynGestServer()
340 CDynGesture* CDynGestServer::FindGesture( const char* name )
342 for( int i = 0; i < m_gestures.size() ; i++ )
344 CString exist_name = m_gestures[i]->GetName();
345 if ( name == exist_name ) return m_gestures[i];
350 CDynGesture* CDynGestServer::AddGesture( const char* name )
352 CDynGesture* new_gest = new CDynGesture( m_num_states, m_num_mix);
353 new_gest->SetName( name );
354 m_gestures.push_back( new_gest );
356 m_fullTrained = FALSE;
358 return m_gestures.back();
361 BOOL CDynGestServer::AddGesture( CDynGesture* new_gest )
363 if ( FindGesture( new_gest->GetName() ) ) return FALSE;
365 m_gestures.push_back( new_gest );
367 m_fullTrained = FALSE;
374 void CDynGestServer::TrainAllGestures()
376 int size = m_gestures.size();
377 for( int i = 0; i < size; i++ )
379 m_gestures[i]->Train();
381 m_fullTrained = TRUE;
385 void CDynGesture::CreateHMM()
387 int* numix = new int[m_num_states];
388 for( int i = 0; i < m_num_states; i++ )
390 numix[i] = m_num_mix;
393 icvCreate1DHMM( &m_hmm, m_num_states, numix, 6 );
397 BOOL CDynGesture::LoadFromFile(const char *obsFilename)
404 BOOL CDynGesture::AddParams( vector<_gr_pose>& input_vect )
406 m_Params.push_back( input_vect );
412 BOOL CDynGesture::SaveParams(const char* path)
419 for ( int i = 0; i < m_Params.size(); i++ )
421 vector<_gr_pose> seq = m_Params[i];
424 filename.Format( "%s\\%s_%03d%s", path, m_name, i,".obs" );
425 file = fopen( filename, "wt");
428 for(int i = 0; i < seq.size(); i++)
430 const fvector& ref = seq[i];
431 CvHuMoments& hu = seq[i].pose;
432 __int64 time = seq[i].time;
433 __int64 time0 = seq[0].time;
435 fprintf(file, "%d %s %f %f %f %f %f %f %f %f %f %f %f %f %f\n",
436 long(time - time0), "*.bmp"/*LPCTSTR(mask_name)*/, ref[0], ref[1], ref[2],
437 ref[3], ref[4], ref[5],
438 (float)hu.hu1, (float)hu.hu2,
439 (float)hu.hu3, (float)hu.hu4,
440 (float)hu.hu5, (float)hu.hu6,
450 BOOL CDynGesture::LoadParams(const char* path)
452 //check if path exist
453 if ( _chdir( path ) < 0 )
464 //construct file name and load
465 filename.Format( "%s_%03d%s", path, i,".obs" );
466 file = fopen( filename, "r");
468 if ( !file ) { end = TRUE; continue; }
471 vector<_gr_pose> positions;
477 pose.resize(6);//construct vector of 6 elements
481 //compute number of observations
482 while(fscanf(file, "%d %s %f %f %f %f %f %f %f %f %f %f %f %f %f\n",
484 &pose[0], &pose[1], &pose[2],
485 &pose[3], &pose[4], &pose[5],
486 &hu[0], &hu[1], &hu[2], &hu[3],
487 &hu[4], &hu[5], &hu[6] )!= EOF)
489 pose.time = int_time;
490 pose.pose.hu1 = hu[0];
491 pose.pose.hu2 = hu[1];
492 pose.pose.hu3 = hu[2];
493 pose.pose.hu4 = hu[3];
494 pose.pose.hu5 = hu[4];
495 pose.pose.hu6 = hu[5];
496 pose.pose.hu7 = hu[6];
498 positions.push_back( pose );
502 m_Params.push_back( positions );
505 if ( i == 0 ) return FALSE;
507 //goto parent directory
513 BOOL CDynGesture::LoadSequence( const char* filename )
516 //construct file name and load
517 file = fopen( filename, "r");
519 if ( !file ) return FALSE;
522 vector<_gr_pose> positions;
528 pose.resize(6);//construct vector of 6 elements
532 //compute number of observations
533 while(fscanf(file, "%d %s %f %f %f %f %f %f %f %f %f %f %f %f %f\n",
535 &pose[0], &pose[1], &pose[2],
536 &pose[3], &pose[4], &pose[5],
537 &hu[0], &hu[1], &hu[2], &hu[3],
538 &hu[4], &hu[5], &hu[6] )!= EOF)
540 pose.time = int_time;
541 pose.pose.hu1 = hu[0];
542 pose.pose.hu2 = hu[1];
543 pose.pose.hu3 = hu[2];
544 pose.pose.hu4 = hu[3];
545 pose.pose.hu5 = hu[4];
546 pose.pose.hu6 = hu[5];
547 pose.pose.hu7 = hu[6];
549 positions.push_back( pose );
553 m_Params.push_back( positions );
560 void CDynGesture::CleanSequences()
562 for( int i = 0; i < m_Sequences.size(); i++ )
564 cvReleaseObsInfo( &m_Sequences[i] );
570 void CDynGesture::ExtractObservations()
572 //clean all observations
575 //extract observations from parameters
576 for( int i = 0; i < m_Params.size(); i++ )
578 vector<_gr_pose>& seq = m_Params[i];
579 int seq_size = seq.size();
580 Cv1DObsInfo* obs_info = NULL;
586 seq_size--; //velocities has 1 less observation than positions
588 obs_info = cvCreateObsInfo( cvSize( seq_size, 1 ), m_vect_size );
590 //construct velocities
591 for( int j = 0; j < seq_size; j++ )
593 fvector& cur_obs = seq[j];
594 fvector& next_obs = seq[j+1];
596 for( int k = 0; k < 3; k++ )
598 obs_info->obs[counter] = 100.f * (next_obs[k] - cur_obs[k]);
605 case OT_VELOCITY_ORIENTATION:
607 seq_size--; //velocities has 1 less observation than positions
609 obs_info = cvCreateObsInfo( cvSize( seq_size, 1 ), m_vect_size );
611 //construct velocities
612 for( int j = 0; j < seq_size; j++ )
614 fvector& cur_obs = seq[j];
615 fvector& next_obs = seq[j+1];
617 for( int k = 0; k < 3; k++ )
619 obs_info->obs[counter] = 100.f * (next_obs[k] - cur_obs[k]);
624 obs_info->obs[counter] = 10.0f * (next_obs[k] - cur_obs[k]);
636 m_Sequences.push_back( obs_info );
641 int CDynGestServer::RecognizeDynGesture(CDynGesture *gesture, float* likelihood)
643 float likelihoodproxy[256]; //256 gestures - maximum
644 if (!likelihood) likelihood = likelihoodproxy;
645 /* recognition stage */
647 int num_gestures = m_gestures.size();
649 for( int i = 0; i < m_gestures.size(); i++ )
651 if ( !m_gestures[i]->m_trained )
653 //AfxMessageBox( "Not all Dynamic Gestures are trained! ", MB_ICONSTOP|MB_OK, 0 );
658 //ASSERT( m_fullTrained );//all stored gestures must be trained
660 float max_like = - FLT_MAX;
661 int recognized = INT_MAX;
663 ASSERT( gesture->GetSeqNumber() == 1 );
665 Cv1DObsInfo** obs_arr = gesture->GetSequences();
666 Cv1DObsInfo* obs = obs_arr[0];
668 for( int h = 0; h < num_gestures; h++ )
670 CvEHMM* hmm = m_gestures[h]->GetHMM();
672 icvEstimate1DObsProb( obs, hmm );
674 likelihood[h] = icvViterbi( obs, hmm );
676 if (likelihood[h] > max_like )
678 max_like = likelihood[h];
683 //printf("%d-th gesture recognized as %d\n", j, image );
687 CvImgObsInfo** CDynGesture::GetSequences()
689 return &(m_Sequences[0]);
692 BOOL CDynGestServer::LoadGestureBase(const char *filename)
697 file = fopen( filename, "r" );
699 if (!file ) return FALSE;
701 char gesture_name[256];
704 while ( fscanf( file, "%s\n", gesture_name )!=EOF )
706 CDynGesture* gesture = new CDynGesture( m_num_states, m_num_mix );
707 gesture->SetName( gesture_name );
708 gesture->LoadParams( gesture_name );
711 if ( ! _chdir( gesture->GetName() ) )
713 if ( gesture->LoadHMM(gesture->GetName() + ".hmm") )
715 gesture->m_trained = TRUE;
716 //change value of m_hmmParams control
717 CStereoGRApp* app = (CStereoGRApp*)AfxGetApp();
718 app->m_hmmParams->m_num_states = gesture->GetHMM()->num_states;
719 app->m_hmmParams->m_num_mix = gesture->GetHMM()->u.state->num_mix;
720 SetHMMParams( app->m_hmmParams->m_num_states,
721 app->m_hmmParams->m_num_mix );
728 AddGesture( gesture );
731 m_loaded_base = filename;
735 BOOL CDynGestServer::SaveGestureBase(const char *name)
738 file = fopen( name, "wt" );
740 for( int i = 0; i < m_gestures.size(); i++ )
742 CDynGesture* gesture = m_gestures[i];
743 fprintf( file, "%s\n", gesture->GetName() );
744 gesture->SaveParams( gesture->GetName() );
745 if ( gesture->m_trained )
747 _chdir( gesture->GetName() );
748 gesture->SaveHMM(gesture->GetName() + ".hmm");
755 m_loaded_base = name;
760 void CDynGestServer::CleanAll()
762 for( int i = 0; i < m_gestures.size(); i++ )
763 delete m_gestures[i];
766 m_fullTrained = false;
770 void CDynGestServer::SetHMMParams( int num_state, int num_mix )
772 if ( m_num_states != num_state || m_num_mix != num_mix )
774 m_num_states = num_state;
776 //change every gesture params
777 for( int i = 0; i < m_gestures.size(); i++ )
779 m_gestures[i]->m_num_mix = num_mix;
780 m_gestures[i]->m_num_states = num_state;
781 m_gestures[i]->m_trained = FALSE;
782 m_gestures[i]->DestroyHMM();
784 m_fullTrained = FALSE;
788 BOOL CDynGestServer::RemoveGesture( const char* name )
790 for( vector<CDynGesture*>::iterator it = m_gestures.begin(); it < m_gestures.end(); it++ )
792 if ( (*it)->GetName() == name )
794 m_gestures.erase( it );
802 void CDynGesture::DestroyHMM()
804 if (m_hmm) icvRelease1DHMM( &m_hmm );
809 void CDynGestServer::DeleteHMMInfo()
811 for( int i = 0; i < m_gestures.size(); i++ )
813 m_gestures[i]->DeleteHMM();
814 if( !_chdir( m_gestures[i]->GetName() ) )
816 remove( m_gestures[i]->GetName() + ".hmm" );
820 m_fullTrained = FALSE;
823 void CDynGesture::DeleteHMM()
825 icvRelease1DHMM( &m_hmm );