hrf-sa-train.h

Go to the documentation of this file.

#pragma once
#include "hrf-ml-train.h"
#include "hrf-corpus.h"
#include <omp.h>


namespace hrf
{
    class SAfunc;
    class SAtrain;

    /* the AIS configurations */
    class AISConfig
    {
    public:
        int nChain; 
        int nInter; 
        AISConfig(int p_nChain = 16, int p_nInter = 1000) :nChain(p_nChain), nInter(p_nInter){}
        void Parse(const char* str) {
            String tempStr(str);
            char *p = strtok(tempStr.GetBuffer(), ":,");
            nChain = atoi(p);
            p = strtok(NULL, ":,");
            nInter = atoi(p);
        }
    };

    /* Save the last sequence of each length in each thread */
    class ThreadData
    {
    public:
        Array<Seq*> aSeqs;
    public:
        ~ThreadData();
        void Create(int maxlen, Model *pModel);
    };
    
    /*
     * \class
     * \brief augment SA training algorithms
    */
    class SAfunc : public MLfunc
    {
        friend class SAtrain;
    protected:
        int m_nMiniBatchSample;   
        int m_nMiniBatchTraining; 
        trf::CorpusRandSelect m_TrainSelect; 
        CorpusCache m_TrainCache; 

        Vec<Prob> m_samplePi; 

    private:
        Vec<double> m_vAllSampleLenCount; 
        Vec<double> m_vCurSampleLenCount; 
        int m_nTotalSample; 

        Vec<double> m_vEmpFeatExp; 
        Vec<double> m_vEmpFeatVar; 
        
        Vec<double> m_vEmpExp; 
        Vec<double> m_vEmpExp2; 
        Vec<double> m_vSampleExp; 
        Vec<double> m_vSampleLen; 

#ifndef _CD
        Array<ThreadData*> m_threadData; 
        Array<Seq*> m_aSeqs; 
#endif
        //Array<trf::RandSeq<int>*> m_trainSelectPerLen; ///< save the index of training sequences of each length


        Mat<double> m_matEmpiricalExp; 
        Mat<double> m_matEmpiricalExp2; 
        Mat<double> m_matSampleExp; 
        Mat<double> m_matSampleLen; 

        /* for emprical variance estimation :*/
#ifdef _Var
        Vec<double> m_vExpValue; 
        Vec<double> m_vExp2Value; 
    public:
        double m_var_gap;
#endif
        //Vec<double> m_vEmpiricalVar; ///< current empirical variance E[f^2]-E[f]^2    

    public:
        AISConfig m_AISConfigForZ; 
        AISConfig m_AISConfigForP; 
        int m_nTrainHiddenSampleTimes; 
        int m_nSampleHiddenSampleTimes; 
        int m_nCDSampleTimes; 
        int m_nSASampleTimes; 
        bool m_bSAMSSample; 
        
    public:
        File m_fdbg;  
        File m_fparm; 
        File m_fgrad; 
        File m_fvar;  
        File m_fexp;  
        File m_fsamp; 
        File m_ftrain;
        File m_feat_mean; 
        File m_feat_var;  

        bool m_bPrintTrain; 
        bool m_bPrintValie; 
        bool m_bPrintTest;  

    public:
        SAfunc() :m_nMiniBatchSample(100), m_nMiniBatchTraining(100) {
#ifdef _Var
            m_var_gap = 1e-5;
#endif
            m_bPrintTrain = true;
            m_bPrintValie = true;
            m_bPrintTest = true;

            m_bSAMSSample = false;
        };
        SAfunc(Model *pModel, CorpusBase *pTrain, CorpusBase *pValid = NULL, CorpusBase *pTest = NULL, int nMinibatch = 100 )
        {
            Reset(pModel, pTrain, pValid, pTest, nMinibatch);
#ifdef _Var
            m_var_gap = 1e-5;
#endif
            m_bPrintTrain = true;
            m_bPrintValie = true;
            m_bPrintTest = true;

            m_bSAMSSample = false;
        }
        ~SAfunc()
        {
            
#ifndef _CD
            for (int i = 0; i < m_aSeqs.GetNum(); i++)
                SAFE_DELETE(m_aSeqs[i]);
            for (int i = 0; i < m_threadData.GetNum(); i++) {
                SAFE_DELETE(m_threadData[i]);
            }
#endif
//          for (int i = 0; i < m_trainSelectPerLen.GetNum(); i++) {
//              SAFE_DELETE(m_trainSelectPerLen[i]);
//          }
        }
        void Reset(Model *pModel, CorpusBase *pTrain, CorpusBase *pValid = NULL, CorpusBase *pTest = NULL, int nMinibatch = 100);
        void PrintInfo();
        int GetNgramFeatNum() const { return m_pModel->m_pFeat->GetNum(); }
        int GetVHmatSize() const { return m_pModel->m_m3dVH.GetSize(); }
        int GetCHmatSize() const { return m_pModel->m_m3dCH.GetSize(); }
        int GetHHmatSize() const { return m_pModel->m_m3dHH.GetSize(); }
//      int GetBiasSize() const { return m_pModel->m_matBias.GetSize(); }
        int GetWeightNum() const { return m_pModel->GetParamNum(); }
        int GetZetaNum() const { return m_pModel->GetMaxLen() + 1; }
        void RandSeq(Seq &seq, int nLen = -1);
        void GetParam(double *pdParams);
        //void GetFeatEmpVar(CorpusBase *pCorpus, Vec<double> &vVar);

        void GetEmpiricalFeatExp(Vec<double> &vExp);
        void GetEmpiricalFeatVar(Vec<double> &vVar);

        int GetEmpiricalExp(VecShell<double> &vExp, VecShell<double> &vExp2, Array<int> &aRandIdx);
        int GetEmpiricalExp(VecShell<double> &vExp, VecShell<double> &vExp2);
        int GetSampleExp(VecShell<double> &vExp, VecShell<double> &vLen);
        void PerfromCD(VecShell<double> &vEmpExp, VecShell<double> &vSamExp, VecShell<double> &vEmpExp2, VecShell<double> &vLen);
        void PerfromSA(VecShell<double> &vEmpExp, VecShell<double> &vSamExp, VecShell<double> &vEmpExp2, VecShell<double> &vLen);
//      void PerfromSAMS(VecShell<double> &vEmpExp, VecShell<double> &vSamExp, VecShell<double> &vEmpExp2, VecShell<double> &vLen);
        /* method = 0 : AIS,   =1 : Chib */
        double GetSampleLL(CorpusBase *pCorpus, int nCalNum = -1, int method = 0);
        void IterEnd(double *pFinalParams);
        void WriteModel(int nEpoch);

        virtual void SetParam(double *pdParams);
        virtual void GetGradient(double *pdGradient);
        virtual double GetValue() { return 0; }
        virtual int GetExtraValues(int t, double *pdValues);
    };

    /*
     * \class
     * \brief Learning rate
    */
    class LearningRate
    {
    public:
        double beta;
        double tc;
        double t0;
    public:
        LearningRate() :beta(1), tc(0), t0(0) {}
        void Reset(const char *pstr, int p_t0);
        double Get(int t);
    };

    /*
     * \class
     * \brief SAtraining
    */
    class SAtrain : public Solve
    {
    protected:
        double m_gamma_lambda;
        double m_gamma_hidden;
        double m_gamma_zeta;

    public:

        LearningRate m_gain_lambda;
        LearningRate m_gain_hidden;
        LearningRate m_gain_zeta;

        bool m_bUpdate_lambda;
        bool m_bUpdate_zeta;

        double m_dir_gap;
        double m_zeta_gap;

        float m_fMomentum; 
        int m_nAvgBeg; 

        double m_fEpochNum; 

        int m_nPrintPerIter;  
        wb::Array<int> m_aWriteAtIter; 

#ifdef _Var
        //double m_var_threshold;
        double m_gamma_var;
        LearningRate m_gain_var;
#endif
    public:
        SAtrain(SAfunc *pfunc = NULL) : Solve(pfunc)
        {
#ifndef _CD
            m_pAlgorithmName = "[SA]";
#else
            m_pAlgorithmName = "[CD]";
#endif

            m_gamma_lambda = 1;
            m_gamma_hidden = 1;
            m_gamma_zeta = 1;


            m_bUpdate_lambda = true;
            m_bUpdate_zeta = true;
            
            m_dir_gap = 0;
            m_zeta_gap = 0;

            m_fMomentum = 0;
            m_nAvgBeg = 0;

            m_fEpochNum = 0;
            m_nPrintPerIter = 1;
// #ifdef _Var
//          m_var_threshold = 1e-4;
// #endif
        }
        virtual bool Run(const double *pInitParams = NULL);
        void UpdateGamma(int nIterNum);
        void UpdateDir(double *pDir, double *pGradient, const double *pParam);
        virtual void Update(double *pdParam, const double *pdDir, double dStep);
        void PrintInfo();
        int CutValue(double *p, int num, double gap);
    };

    
}