[irstlm] 06/78: added functions to only compute the lm state based on the index of ngram
Giulio Paci
giuliopaci-guest at moszumanska.debian.org
Tue May 17 07:47:00 UTC 2016
This is an automated email from the git hooks/post-receive script.
giuliopaci-guest pushed a commit to tag adaptiveLM.v0.10
in repository irstlm.
commit faec6449ca01b80a4c1d8215137ce7313946b10b
Author: Nicola Bertoldi <bertoldi at fbk.eu>
Date: Fri Nov 6 17:01:07 2015 +0100
added functions to only compute the lm state based on the index of ngram
---
src/lmInterpolation.cpp | 111 +++++++++++++++++++++++++++++++++++++++++++++---
src/lmInterpolation.h | 6 +++
2 files changed, 112 insertions(+), 5 deletions(-)
diff --git a/src/lmInterpolation.cpp b/src/lmInterpolation.cpp
index 33aaa21..113c18b 100644
--- a/src/lmInterpolation.cpp
+++ b/src/lmInterpolation.cpp
@@ -140,7 +140,7 @@ namespace irstlm {
}
//return log10 prob of an ngram
-// double lmInterpolation::clprob(ngram ng, double* bow,int* bol,char** maxsuffptr,unsigned int* statesize,bool* extendible)
+ // double lmInterpolation::clprob(ngram ng, double* bow,int* bol,char** maxsuffptr,unsigned int* statesize,bool* extendible)
double lmInterpolation::clprob(ngram ng, double* bow,int* bol,ngram_state_t* maxsuffidx, char** maxsuffptr,unsigned int* statesize,bool* extendible)
{
@@ -155,14 +155,14 @@ namespace irstlm {
bool _extendible=false;
bool actualextendible=false;
-// ngram_state_t* maxsuffidx = new ngram_state_t;
+ // ngram_state_t* maxsuffidx = new ngram_state_t;
for (size_t i=0; i<m_lm.size(); i++) {
if (m_weight[i]>0.0){
ngram _ng(m_lm[i]->getDict());
_ng.trans(ng);
-// _logpr=m_lm[i]->clprob(_ng,&_bow,&_bol,&_maxsuffptr,&_statesize,&_extendible);
+ // _logpr=m_lm[i]->clprob(_ng,&_bow,&_bol,&_maxsuffptr,&_statesize,&_extendible);
_logpr=m_lm[i]->clprob(_ng,&_bow,&_bol,&_maxsuffidx,&_maxsuffptr,&_statesize,&_extendible);
IFVERBOSE(3){
@@ -218,7 +218,7 @@ namespace irstlm {
return log10(pr);
}
-// double lmInterpolation::clprob(int* codes, int sz, double* bow,int* bol,char** maxsuffptr,unsigned int* statesize,bool* extendible)
+ // double lmInterpolation::clprob(int* codes, int sz, double* bow,int* bol,char** maxsuffptr,unsigned int* statesize,bool* extendible)
double lmInterpolation::clprob(int* codes, int sz, double* bow,int* bol,ngram_state_t* maxsuffidx,char** maxsuffptr,unsigned int* statesize,bool* extendible)
{
@@ -227,10 +227,110 @@ namespace irstlm {
ong.pushc(codes,sz);
MY_ASSERT (ong.size == sz);
-// return clprob(ong, bow, bol, maxsuffptr, statesize, extendible);
+ // return clprob(ong, bow, bol, maxsuffptr, statesize, extendible);
return clprob(ong, bow, bol, maxsuffidx, maxsuffptr, statesize, extendible);
}
+ const char *lmInterpolation::cmaxsuffptr(ngram ng, unsigned int* statesize){
+
+ char *maxsuffptr=NULL;
+ unsigned int _statesize=0,actualstatesize=0;
+
+ // ngram_state_t* maxsuffidx = new ngram_state_t;
+
+ for (size_t i=0; i<m_lm.size(); i++) {
+
+ if (m_weight[i]>0.0){
+ ngram _ng(m_lm[i]->getDict());
+ _ng.trans(ng);
+
+ const char* _maxsuffptr = m_lm[i]->cmaxsuffptr(_ng,&_statesize);
+
+ IFVERBOSE(3){
+ //cerr.precision(10);
+ VERBOSE(3," LM " << i << " weight:" << m_weight[i] << std::endl);
+ VERBOSE(3," _statesize:" << _statesize << std::endl);
+ }
+
+ /*
+ //TO CHECK the following claims
+ //What is the statesize of a LM interpolation? The largest _statesize among the submodels
+ //What is the maxsuffptr of a LM interpolation? The _maxsuffptr of the submodel with the largest _statesize
+ */
+
+ if(_statesize > actualstatesize || i == 0) {
+ maxsuffptr = (char*) _maxsuffptr;
+ actualstatesize = _statesize;
+ }
+ }
+ }
+ if (statesize) *statesize=actualstatesize;
+
+ if (statesize) VERBOSE(3, " statesize:" << *statesize << std::endl);
+
+ return maxsuffptr;
+ }
+
+ const char *lmInterpolation::cmaxsuffptr(int* codes, int sz, unsigned int* statesize)
+ {
+ //create the actual ngram
+ ngram ong(dict);
+ ong.pushc(codes,sz);
+ MY_ASSERT (ong.size == sz);
+ return cmaxsuffptr(ong, statesize);
+ }
+
+ ngram_state_t lmInterpolation::cmaxsuffidx(ngram ng, unsigned int* statesize)
+ {
+ ngram_state_t maxsuffidx=0;
+ unsigned int _statesize=0,actualstatesize=0;
+
+ // ngram_state_t* maxsuffidx = new ngram_state_t;
+
+ for (size_t i=0; i<m_lm.size(); i++) {
+
+ if (m_weight[i]>0.0){
+ ngram _ng(m_lm[i]->getDict());
+ _ng.trans(ng);
+
+ ngram_state_t _maxsuffidx = m_lm[i]->cmaxsuffidx(_ng,&_statesize);
+
+ IFVERBOSE(3){
+ //cerr.precision(10);
+ VERBOSE(3," LM " << i << " weight:" << m_weight[i] << std::endl);
+ VERBOSE(3," _statesize:" << _statesize << std::endl);
+ }
+
+ /*
+ //TO CHECK the following claims
+ //What is the statesize of a LM interpolation? The largest _statesize among the submodels
+ //What is the maxsuffptr of a LM interpolation? The _maxsuffptr of the submodel with the largest _statesize
+ */
+
+ if(_statesize > actualstatesize || i == 0) {
+ maxsuffidx = _maxsuffidx;
+ actualstatesize = _statesize;
+ }
+ }
+ }
+
+ if (statesize) *statesize=actualstatesize;
+
+ if (statesize) VERBOSE(3, " statesize:" << *statesize << std::endl);
+
+ return maxsuffidx;
+ }
+
+ ngram_state_t lmInterpolation::cmaxsuffidx(int* codes, int sz, unsigned int* statesize)
+ {
+ //create the actual ngram
+ ngram ong(dict);
+ ong.pushc(codes,sz);
+ MY_ASSERT (ong.size == sz);
+ return cmaxsuffidx(ong, statesize);
+ }
+
+
double lmInterpolation::setlogOOVpenalty(int dub)
{
MY_ASSERT(dub > dict->size());
@@ -249,3 +349,4 @@ namespace irstlm {
return logOOVpenalty;
}
}//namespace irstlm
+
diff --git a/src/lmInterpolation.h b/src/lmInterpolation.h
index 7219a5c..9b260d8 100644
--- a/src/lmInterpolation.h
+++ b/src/lmInterpolation.h
@@ -77,6 +77,12 @@ public:
virtual double clprob(ngram ng, double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL);
virtual double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL);
+
+ virtual const char *cmaxsuffptr(ngram ong, unsigned int* size=NULL);
+ virtual const char *cmaxsuffptr(int* codes, int sz, unsigned int* size=NULL);
+ virtual ngram_state_t cmaxsuffidx(ngram ong, unsigned int* size=NULL);
+ virtual ngram_state_t cmaxsuffidx(int* codes, int sz, unsigned int* size=NULL);
+
int maxlevel() const {
return maxlev;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/irstlm.git
More information about the debian-science-commits
mailing list