trf-corpus.h

Go to the documentation of this file.

// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright 2014-2015 Tsinghua University
// Author: wb.th08@gmail.com (Bin Wang), ozj@tsinghua.edu.cn (Zhijian Ou) 
//
// All h, cpp, cc, and script files (e.g. bat, sh, pl, py) should include the above 
// license declaration. Different coding language may use different comment styles.


#pragma once
#include "trf-vocab.h"
#include "trf-model.h"

namespace trf
{
    class CorpusBase
    {
    protected:
        String m_filename;

        int m_nMinLen; 
        int m_nMaxLen; 
        int m_nNum; 
    public:
        CorpusBase() : m_nNum(0), m_nMinLen(0), m_nMaxLen(0) {};

        virtual void Reset(const char *pfilename) = 0;
        virtual bool GetSeq(int nLine, Array<VocabID> &aSeq) = 0;
        virtual void GetLenCount(Array<int> &aLenCount) = 0;

        virtual int GetNum() const { return m_nNum; }
        virtual int GetMinLen() const { return m_nMinLen; }
        virtual int GetMaxLen() const { return m_nMaxLen; }
        const char* GetFileName() const { return m_filename.GetBuffer();  }
    };

    class CorpusTxt : public CorpusBase
    {
    protected:
        Array<Array<VocabID>*> m_aSeq;
    public:
        CorpusTxt() {};
        CorpusTxt(const char *pfilename) { Reset(pfilename); }
        ~CorpusTxt();
        virtual void Reset(const char *pfilename);
        virtual bool GetSeq(int nLine, Array<VocabID> &aSeq);
        virtual void GetLenCount(Array<int> &aLenCount);
    };

    class CorpusRandSelect
    {
    protected:
        CorpusBase *m_pCorpus;

        Array<int> m_aRandIdx;
        int m_nCurIdx;
    public:
        CorpusRandSelect() :m_pCorpus(NULL) {}
        CorpusRandSelect(CorpusBase *pCorpus) { Reset(pCorpus); }
        void Reset(CorpusBase *p);
        void GetSeq(Array<VocabID> &aSeq);
        void GetIdx(int *pIdx, int nNum);
        void RandomIdx(int nNum);
    };

    template <class Type>
    class RandSeq
    {
    protected:
        Array<Type> m_aSeq;
        int m_nCur;
    public:
        RandSeq() :m_nCur(0) {}
        RandSeq(Array<Type> &a) : m_nCur(0) { m_aSeq.Copy(a); }
        void Random() {
            int len = m_aSeq.GetNum();
            for (int i = 0; i < len; i++) {
                int s = rand() % (len - i);
                //exchange i+s and i positions
                int temp = m_aSeq[i + s];
                m_aSeq[i + s] = m_aSeq[i];
                m_aSeq[i] = temp;
            }
            m_nCur = 0; // reset the current position
        }
        void Add(Type t) { m_aSeq.Add(t); }
        Type Get() { 
            if (m_nCur >= m_aSeq.GetNum()) {
                Random();
            }
            return m_aSeq[m_nCur++];
        }
    };
}