wb-word-cluster.h

Go to the documentation of this file.

// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright 2014-2015 Tsinghua University
// Author: wb.th08@gmail.com (Bin Wang), ozj@tsinghua.edu.cn (Zhijian Ou)
//
// All h, cpp, cc, and script files (e.g. bat, sh, pl, py) should include the above
// license declaration. Different coding language may use different comment styles.

#ifndef _WB_WORD_CLUSTER_H_
#define _WB_WORD_CLUSTER_H_

#include "wb-system.h"

namespace wb
{
    class WordCluster
    {
    public:
        LHash<int, int> m_wordCount;  
        LHash<int, int> m_classCount; 
        Trie<int, int> m_wordGramCount;  
        Trie<int, int> m_invWordGram; 
        //wbTrie<int, int> m_classGramCount; ///< N(g_w,g_v);
        int **m_pClassGramCount; 
        Trie<int, int> m_wordClassCount; 
        Trie<int, int> m_classWordCount; 

        double m_dWordLogSum; 

        Array<int> m_aClass; 
        int m_nClassNum;
        int m_nVocabSize; 
        int m_nSentNum; 

        int m_nUnigramNum;
        int m_nBigramNum;

        char *m_pathWordClass;
        char *m_pathClassWord;
        char *m_pathTagVocab;


    public:
        WordCluster(int nClass) : m_nClassNum(nClass){
            SAFE_NEW_DARRAY(m_pClassGramCount, int, nClass + 1, nClass + 1);

            m_pathWordClass = NULL;
            m_pathClassWord = NULL;
            m_pathTagVocab = NULL;
        };
        ~WordCluster(void) {
            SAFE_DELETE_DARRAY(m_pClassGramCount, m_nClassNum + 1);
        };

        void Reverse(int *pGram) { int n = pGram[0]; pGram[0] = pGram[1]; pGram[1] = n; }
        void InitCount(const char *path, const char *pTagVocab = NULL);
        void UpdataCount();
        void CountAdd(LHash<int, int> &count, int nWord, int nAdd) {
            bool bFound;
            int *pCount = count.Insert(nWord, bFound);
            if (!bFound) *pCount = nAdd;
            else *pCount += nAdd;
        }
        void CountAdd(Trie<int, int> &count, int *pWord, int nLen, int nAdd) {
            bool bFound;
            int *pCount = count.Insert(pWord, nLen, bFound);
            if (!bFound) *pCount = nAdd;
            else *pCount += nAdd;
        }
        void CountAdd(int **pCount, int *pWord, int nLen, int nAdd) {
            pCount[pWord[0]][pWord[1]] += nAdd;
        }
        void WriteCount(LHash<int, int> &count, File &file);
        void WriteCount(Trie<int, int> &count, File &file, bool bReverse = false);
        void WriteRes_WordClass(const char *path);
        void WriteRes_ClassWord(const char *path);
        void WriteRes_TagVocab(const char *path);
        void Read_TagVocab(const char *path);

        double LogLikelihood();
        void MoveWord(int nWord, bool bOut = true);
        void ExchangeWord(int nWord, int nToClass);

        void Cluster(int nMaxTime = -1);

        void SimpleCluster();
    };

    class WordCluster_t
    {
    public:
        LHash<int, int> m_word_count;       
        Trie<int, int> m_wgram_count;   
        Trie<int, int> m_inv_wgram_count;       

        LHash<int, int> m_class;            
        Trie<int, int> m_class_gram;        
        Trie<int, int> m_word_class_gram;   
        Trie<int, int> m_class_word_gram;   

        Mat<int> m_tCountBuf;       
        Mat<int> m_tMap;            

        Array<int> m_mCountBuf;     
        Array<int> m_mMap;          


        int m_nClassNum;    
        int m_nVocabSize;   
        int m_nSentNum;     

        int m_nUnigramNum;
        int m_nBigramNum;
        double m_dWordLogSum;

        String m_pathRes;       

//      char *m_pathWordClass;
//      char *m_pathClassWord;
//      char *m_pathTagVocab;

        //WordCluster cluster;

    public:
        WordCluster_t(int nClass, char *pathRes = NULL):
            //cluster(nClass),
            m_nClassNum(nClass){
//          m_pathWordClass = NULL;
//          m_pathClassWord = NULL;
//          m_pathTagVocab = NULL;
            if (pathRes == NULL) {
                m_pathRes = "word_cluster.default.res";
            }
            else {
                m_pathRes = pathRes;
            }
        };
        ~WordCluster_t(void) {
        };
        void WriteRes(const char *path);
        void ReadRes(const char *path);
        void Reverse(int *pGram) { int n = pGram[0]; pGram[0] = pGram[1]; pGram[1] = n; }
        void InitCount(const char *path, const char *path_init_res = NULL);
        void UpdateCount(Array<int> &aCountBuf);
        void CountAdd(Array<int> &aCountBuf, LHash<int, int> &hash, int key, int count);
        void CountAdd(Array<int> &aCountBuf, Trie<int, int> &hash, int *pKey, int nLen, int count);
        void CountAdd(VecShell<int> &aCountBuf, LHash<int, int> &hash, int key, int count);
        void CountAdd(VecShell<int> &aCountBuf, Trie<int, int> &hash, int *pKey, int nLen, int count);

        void CopyCountToThreads(Array<int> &aCountBuf);
        void MoveWord(VecShell<int> vCountBuf, VecShell<int> vMap, int nWord, bool bOut = true);
        void ExchangeWord(VecShell<int> vCountBuf, VecShell<int> vMap, int nWord, int nToClass);
        void Cluster(int nMaxTime = -1);
        double LogLikelihood(VecShell<int> vCountBuf);

        void SimpleCluster();
    };s
}

#endif