Tesseract 3.02

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
char_bigrams.cpp
Go to the documentation of this file.
1 /**********************************************************************
2 * File: char_bigrams.cpp
3 * Description: Implementation of a Character Bigrams Class
4 * Author: Ahmad Abdulkader
5 * Created: 2007
6 *
7 * (C) Copyright 2008, Google Inc.
8 ** Licensed under the Apache License, Version 2.0 (the "License");
9 ** you may not use this file except in compliance with the License.
10 ** You may obtain a copy of the License at
11 ** http://www.apache.org/licenses/LICENSE-2.0
12 ** Unless required by applicable law or agreed to in writing, software
13 ** distributed under the License is distributed on an "AS IS" BASIS,
14 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 ** See the License for the specific language governing permissions and
16 ** limitations under the License.
17 *
18 **********************************************************************/
19
20 #include <algorithm>
21 #include <math.h>
22 #include <string>
23 #include <vector>
24
25 #include "char_bigrams.h"
26 #include "cube_utils.h"
27 #include "ndminx.h"
28 #include "cube_const.h"
29
30 namespace tesseract {
31
32 CharBigrams::CharBigrams() {
33 memset(&bigram_table_, 0, sizeof(bigram_table_));
34 }
35
36 CharBigrams::~CharBigrams() {
37 if (bigram_table_.char_bigram != NULL) {
38 for (int ch1 = 0; ch1 <= bigram_table_.max_char; ch1++) {
39 CharBigram *char_bigram = bigram_table_.char_bigram + ch1;
40
41 if (char_bigram->bigram != NULL) {
42 delete []char_bigram->bigram;
43 }
44 }
45 delete []bigram_table_.char_bigram;
46 }
47 }
48
49 CharBigrams *CharBigrams::Create(const string &data_file_path,
50 const string &lang) {
51 string file_name;
52 string str;
53
54 file_name = data_file_path + lang;
55 file_name += ".cube.bigrams";
56
57 // load the string into memory
58 if (!CubeUtils::ReadFileToString(file_name, &str)) {
59 return NULL;
60 }
61
62 // construct a new object
63 CharBigrams *char_bigrams_obj = new CharBigrams();
64 if (char_bigrams_obj == NULL) {
65 fprintf(stderr, "Cube ERROR (CharBigrams::Create): could not create "
66 "character bigrams object.\n");
67 return NULL;
68 }
69 CharBigramTable *table = &char_bigrams_obj->bigram_table_;
70
71 table->total_cnt = 0;
72 table->max_char = -1;
73 table->char_bigram = NULL;
74
75 // split into lines
76 vector<string> str_vec;
77 CubeUtils::SplitStringUsing(str, "\r\n", &str_vec);
78
79 for (int big = 0; big < str_vec.size(); big++) {
80 char_32 ch1;
81 char_32 ch2;
82 int cnt;
83 if (sscanf(str_vec[big].c_str(), "%d %x %x", &cnt, &ch1, &ch2) != 3) {
84 fprintf(stderr, "Cube ERROR (CharBigrams::Create): invalid format "
85 "reading line: %s\n", str_vec[big].c_str());
86 return NULL;
87 }
88
89 // expand the bigram table
90 if (ch1 > table->max_char) {
91 CharBigram *char_bigram = new CharBigram[ch1 + 1];
92 if (char_bigram == NULL) {
93 fprintf(stderr, "Cube ERROR (CharBigrams::Create): error allocating "
94 "additional memory for character bigram table.\n");
95 return NULL;
96 }
97
98 if (table->char_bigram != NULL && table->max_char >= 0) {
99 memcpy(char_bigram, table->char_bigram,
100 (table->max_char + 1) * sizeof(*char_bigram));
101
102 delete []table->char_bigram;
103 }
104 table->char_bigram = char_bigram;
105
106 // init
107 for (int new_big = table->max_char + 1; new_big <= ch1; new_big++) {
108 table->char_bigram[new_big].total_cnt = 0;
109 table->char_bigram[new_big].max_char = -1;
110 table->char_bigram[new_big].bigram = NULL;
111 }
112 table->max_char = ch1;
113 }
114
115 if (ch2 > table->char_bigram[ch1].max_char) {
116 Bigram *bigram = new Bigram[ch2 + 1];
117 if (bigram == NULL) {
118 fprintf(stderr, "Cube ERROR (CharBigrams::Create): error allocating "
119 "memory for bigram.\n");
120 return NULL;
121 }
122
123 if (table->char_bigram[ch1].bigram != NULL &&
124 table->char_bigram[ch1].max_char >= 0) {
125 memcpy(bigram, table->char_bigram[ch1].bigram,
126 (table->char_bigram[ch1].max_char + 1) * sizeof(*bigram));
127 delete []table->char_bigram[ch1].bigram;
128 }
129 table->char_bigram[ch1].bigram = bigram;
130
131 // init
132 for (int new_big = table->char_bigram[ch1].max_char + 1;
133 new_big <= ch2; new_big++) {
134 table->char_bigram[ch1].bigram[new_big].cnt = 0;
135 }
136 table->char_bigram[ch1].max_char = ch2;
137 }
138
139 table->char_bigram[ch1].bigram[ch2].cnt = cnt;
140 table->char_bigram[ch1].total_cnt += cnt;
141 table->total_cnt += cnt;
142 }
143
144 // compute costs (-log probs)
145 table->worst_cost = static_cast<int>(
146 -PROB2COST_SCALE * log(0.5 / table->total_cnt));
147 for (char_32 ch1 = 0; ch1 <= table->max_char; ch1++) {
148 for (char_32 ch2 = 0; ch2 <= table->char_bigram[ch1].max_char; ch2++) {
149 int cnt = table->char_bigram[ch1].bigram[ch2].cnt;
150 table->char_bigram[ch1].bigram[ch2].cost =
151 static_cast<int>(-PROB2COST_SCALE *
152 log(MAX(0.5, static_cast<double>(cnt)) /
153 table->total_cnt));
154 }
155 }
156 return char_bigrams_obj;
157 }
158
159 int CharBigrams::PairCost(char_32 ch1, char_32 ch2) const {
160 if (ch1 > bigram_table_.max_char) {
161 return bigram_table_.worst_cost;
162 }
163 if (ch2 > bigram_table_.char_bigram[ch1].max_char) {
164 return bigram_table_.worst_cost;
165 }
166 return bigram_table_.char_bigram[ch1].bigram[ch2].cost;
167 }
168
169 int CharBigrams::Cost(const char_32 *char_32_ptr, CharSet *char_set) const {
170 if (!char_32_ptr || char_32_ptr[0] == 0) {
171 return bigram_table_.worst_cost;
172 }
173 int cost = MeanCostWithSpaces(char_32_ptr);
174 if (CubeUtils::StrLen(char_32_ptr) >= kMinLengthCaseInvariant &&
175 CubeUtils::IsCaseInvariant(char_32_ptr, char_set)) {
176 char_32 *lower_32 = CubeUtils::ToLower(char_32_ptr, char_set);
177 if (lower_32 && lower_32[0] != 0) {
178 int cost_lower = MeanCostWithSpaces(lower_32);
179 cost = MIN(cost, cost_lower);
180 delete [] lower_32;
181 }
182 char_32 *upper_32 = CubeUtils::ToUpper(char_32_ptr, char_set);
183 if (upper_32 && upper_32[0] != 0) {
184 int cost_upper = MeanCostWithSpaces(upper_32);
185 cost = MIN(cost, cost_upper);
186 delete [] upper_32;
187 }
188 }
189 return cost;
190 }
191
192 int CharBigrams::MeanCostWithSpaces(const char_32 *char_32_ptr) const {
193 if (!char_32_ptr)
194 return bigram_table_.worst_cost;
195 int len = CubeUtils::StrLen(char_32_ptr);
196 int cost = 0;
197 int c = 0;
198 cost = PairCost(' ', char_32_ptr[0]);
199 for (c = 1; c < len; c++) {
200 cost += PairCost(char_32_ptr[c - 1], char_32_ptr[c]);
201 }
202 cost += PairCost(char_32_ptr[len - 1], ' ');
203 return static_cast<int>(cost / static_cast<double>(len + 1));
204 }
205 } // namespace tesseract