trf-sa-train.h

Go to the documentation of this file.

// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright 2014-2015 Tsinghua University
// Author: wb.th08@gmail.com (Bin Wang), ozj@tsinghua.edu.cn (Zhijian Ou)
//
// All h, cpp, cc, and script files (e.g. bat, sh, pl, py) should include the above
// license declaration. Different coding language may use different comment styles.


#pragma once
#include "trf-ml-train.h"
#include <omp.h>


namespace trf
{
    class SAfunc;
    class SAtrain;

    /* the AIS configurations */
    class AISConfig
    {
    public:
        int nChain; 
        int nInter; 
        AISConfig(int p_nChain = 16, int p_nInter = 1000) :nChain(p_nChain), nInter(p_nInter){}
        void Parse(const char* str) {
            String tempStr(str);
            char *p = strtok(tempStr.GetBuffer(), ":,");
            nChain = atoi(p);
            p = strtok(NULL, ":,");
            nInter = atoi(p);
        }
    };

    /* Save the last sequence of each length in each thread */
    class ThreadData
    {
    public:
        Array<Seq*> aSeqs;
    public:
        ~ThreadData();
        void Create(int maxlen, Model *pModel);
    };


    /*
     * \class
     * \brief augment SA training algorithms
    */
    class SAfunc : public MLfunc
    {
        friend class SAtrain;
    protected:
        int m_nMiniBatchSample;  

        Vec<Prob> m_samplePi; 

        Vec<double> m_vAllSampleLenCount; 
        Vec<double> m_vCurSampleLenCount; 
        int m_nTotalSample; 

//      Vec<double> m_vEmpiricalExp; ///< the empirical expectation
//      Vec<double> m_vEmpiricalExp2; ///< the empirical expectation E[f^2]
        Vec<double> m_vSampleExp; 
        Vec<double> m_vSampleExp2; 
        Vec<double> m_vSampleLen; 


        //Array<ThreadData*> m_threadData; ///< save the last sequence of each threads
        Array<Seq*> m_threadSeq; 

//      Mat<double> m_matEmpiricalExp; ///< the empirical expectation of each thread
//      Mat<double> m_matEmpiricalExp2; ///< empirical E[f^2] of each thread
        Mat<double> m_matSampleExp; 
        Mat<double> m_matSampleExp2; 
        Mat<double> m_matSampleLen; 

        Vec<double> m_vEmpiricalVar; 

    public:
        double m_fRegL2; 
        double m_var_gap; 

        AISConfig m_AISConfigForZ; 
        AISConfig m_AISConfigForP; 
        int m_nCDSampleTimes; 
        int m_nSASampleTimes; 

    public:
        File m_fdbg;  
        File m_fparm; 
        File m_fgrad; 
        File m_fmean; 
        File m_fvar;  
        File m_fexp;  
        File m_fsamp; 
        File m_ftrain;
        File m_ftrainLL; 
        File m_fvallidLL;
        File m_ftestLL;  


    public:
        SAfunc() :m_nMiniBatchSample(100) {
            m_var_gap = 1e-15;
            m_fRegL2 = 0;
        };
        SAfunc(Model *pModel, CorpusBase *pTrain, CorpusBase *pValid = NULL, CorpusBase *pTest = NULL, int nMinibatch = 100 )
        {
            m_var_gap = 1e-15;
            m_fRegL2 = 0;
            Reset(pModel, pTrain, pValid, pTest, nMinibatch);
        }
        ~SAfunc()
        {
#ifndef _CD
            for (int i = 0; i < m_threadSeq.GetNum(); i++) {
                SAFE_DELETE(m_threadSeq[i]);
            }
#endif
        }
        virtual void Reset(Model *pModel, CorpusBase *pTrain, CorpusBase *pValid = NULL, CorpusBase *pTest = NULL, int nMinibatch = 100);
        void PrintInfo();
        int GetFeatNum() const { return m_pModel->GetParamNum(); }
        int GetZetaNum() const { return m_pModel->GetMaxLen() + 1; }
        void RandSeq(Seq &seq, int nLen = -1);
        void GetParam(double *pdParams);
        void GetEmpVar(CorpusBase *pCorpus, Vec<double> &vVar);

        virtual void GetSampleExp(VecShell<double> &vExp, VecShell<double> &vExp2, VecShell<double> &vLen);

        void IterEnd(double *pFinalParams);
        void WriteModel(int nEpoch);

        virtual void SetParam(double *pdParams);
        virtual void GetGradient(double *pdGradient);
        virtual double GetValue() { return 0; }
        virtual int GetExtraValues(int t, double *pdValues);
    };

    /*
     * \class
     * \brief Learning rate
    */
    class LearningRate
    {
    public:
        double beta;
        double tc;
        double t0;
    public:
        LearningRate() :beta(1), tc(0), t0(0) {}
        void Reset(const char *pstr, int p_t0);
        double Get(int t);
    };

    /*
     * \class
     * \brief SAtraining
    */
    class SAtrain : public Solve
    {
    protected:
        double m_gamma_lambda;
        double m_gamma_zeta;

    public:
        LearningRate m_gain_lambda;
        LearningRate m_gain_zeta;
        //double m_zeta_upgap; ///< the gap for zeta update

        bool m_bUpdate_lambda;
        bool m_bUpdate_zeta;
        double m_dir_gap; 

        int m_nAvgBeg; 
        float m_fEpochNun; 
        int m_nPrintPerIter;  
        wb::Array<int> m_aWriteAtIter; 

#ifdef _Adam
        double adam_beta1;
        double adam_beta2;
        double adam_sigma;
        double adam_alpha;
        Vec<double> adam_m; 
        Vec<double> adam_v; 
#endif

#ifdef _Hession
        Vec<double> m_avgHes; 
#endif

    public:
        SAtrain(SAfunc *pfunc = NULL) : Solve(pfunc)
        {
            m_pAlgorithmName = "[SAMS]";

            m_gamma_lambda = 1;
            m_gamma_zeta = 1;
            //m_zeta_upgap = 10;


            m_bUpdate_lambda = true;
            m_bUpdate_zeta = true;
            m_dir_gap = 1.0;

            m_nAvgBeg = 0;

            m_fEpochNun = 0;
            m_nPrintPerIter = 1;
#ifdef _Hession
            m_avgHes.Reset(pfunc->GetFeatNum());
            m_avgHes.Fill(0);
#endif
#ifdef _Adam
            adam_beta1 = 0.9;
            adam_beta2 = 0.999;
            adam_alpha = 1e-3;
            adam_sigma = 1e-8;
            adam_m.Reset(pfunc->GetFeatNum());
            adam_v.Reset(pfunc->GetFeatNum());
            adam_m.Fill(0);
            adam_v.Fill(0);
#endif
        }
        virtual bool Run(const double *pInitParams = NULL);
        void UpdateGamma(int nIterNum);
        void UpdateDir(double *pDir, double *pGradient, const double *pParam);
        virtual void Update(double *pdParam, const double *pdDir, double dStep);
        void PrintInfo();
    };

}