42 #include "EST_String.h"
43 #include "EST_Token.h"
44 #include "EST_error.h"
45 #include "EST_Ngrammar.h"
53 if ((input_format ==
"sentence_per_line") ||
54 (input_format ==
"sentence_per_file"))
58 window[ngram.order()-1] = t;
59 if (ngram.wordlist_index(t) == -1)
60 cerr <<
"EST_Ngrammar test: skipping bad word \"" <<
63 else if (input_format ==
"ngram_per_line")
65 for (i=0; i < ngram.order(); i++)
69 if (ngram.wordlist_index(t) == -1)
70 cerr <<
"EST_Ngrammar test: skipping bad word \"" <<
75 EST_error(
"EST_Ngrammar test: unknown input format \"%s\"\n",
76 (
const char *)input_format);
81 for (i=0; i < ngram.order(); i++)
82 if (ngram.wordlist_index(window(i)) == -1)
110 ts.
open(stdin,FALSE);
111 else if (ts.
open(filename) == -1)
112 EST_error(
"EST_Ngrammar test: unable to open test file \"%s\"\n",
113 (
const char *)filename);
117 ngram.fill_window_start(window,prev,prev_prev);
120 (get_next_window(ts,window,input_format,ngram) == TRUE))
122 prob = ngram.probability(window);
125 if ((input_format ==
"sentence_per_line") && (ts.
eoln()))
126 ngram.fill_window_start(window,prev,prev_prev);
130 raw_entropy = -1 * H;
131 entropy = -1 * (H/Q);
132 perplexity = pow(2.0,entropy);