Tesseract 3.02

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ambigs.h
Go to the documentation of this file.
1
2 // File: ambigs.h
3 // Description: Constants, flags, functions for dealing with
4 // ambiguities (training and recognition).
5 // Author: Daria Antonova
6 // Created: Mon Aug 23 11:26:43 PDT 2008
7 //
8 // (C) Copyright 2008, Google Inc.
9 // Licensed under the Apache License, Version 2.0 (the "License");
10 // you may not use this file except in compliance with the License.
11 // You may obtain a copy of the License at
12 // http://www.apache.org/licenses/LICENSE-2.0
13 // Unless required by applicable law or agreed to in writing, software
14 // distributed under the License is distributed on an "AS IS" BASIS,
15 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 // See the License for the specific language governing permissions and
17 // limitations under the License.
18 //
20
21 #ifndef TESSERACT_CCUTIL_AMBIGS_H_
22 #define TESSERACT_CCUTIL_AMBIGS_H_
23
24 #include "elst.h"
25 #include "tprintf.h"
26 #include "unichar.h"
27 #include "unicharset.h"
28 #include "genericvector.h"
29
30 #define MAX_AMBIG_SIZE 10
31
32 namespace tesseract {
33
34 typedef GenericVector<UNICHAR_ID> UnicharIdVector;
35
36 static const int kUnigramAmbigsBufferSize = 1000;
37 static const char kAmbigNgramSeparator[] = { ' ', '\0' };
38 static const char kAmbigDelimiters[] = "\t ";
39 static const char kIllegalMsg[] =
40 "Illegal ambiguity specification on line %d\n";
41 static const char kIllegalUnicharMsg[] =
42 "Illegal unichar %s in ambiguity specification\n";
43
44 enum AmbigType {
45 NOT_AMBIG, // the ngram pair is not ambiguous
46 REPLACE_AMBIG, // ocred ngram should always be substituted with correct
47 DEFINITE_AMBIG, // add correct ngram to the classifier results (1-1)
48 SIMILAR_AMBIG, // use pairwise classifier for ocred/correct pair (1-1)
49 CASE_AMBIG, // this is a case ambiguity (1-1)
50
51 AMBIG_TYPE_COUNT // number of enum entries
52 };
53
54 // A collection of utility functions for arrays of UNICHAR_IDs that are
55 // terminated by INVALID_UNICHAR_ID.
56 class UnicharIdArrayUtils {
57 public:
58 // Compares two arrays of unichar ids. Returns -1 if the length of array1 is
59 // less than length of array2, if any array1[i] is less than array2[i].
60 // Returns 0 if the arrays are equal, 1 otherwise.
61 // The function assumes that the arrays are terminated by INVALID_UNICHAR_ID.
62 static inline int compare(const UNICHAR_ID array1[],
63 const UNICHAR_ID array2[]) {
64 const UNICHAR_ID *ptr1 = array1;
65 const UNICHAR_ID *ptr2 = array2;
66 while (*ptr1 != INVALID_UNICHAR_ID && *ptr2 != INVALID_UNICHAR_ID) {
67 if (*ptr1 != *ptr2) return *ptr1 < *ptr2 ? -1 : 1;
68 ++ptr1;
69 ++ptr2;
70 }
71 if (*ptr1 == INVALID_UNICHAR_ID && *ptr2 == INVALID_UNICHAR_ID) return 0;
72 return *ptr1 == INVALID_UNICHAR_ID ? -1 : 1;
73 }
74
75 // Look uid in the vector of uids. If found, the index of the matched
76 // element is returned. Otherwise, it returns -1.
77 static inline int find_in(const UnicharIdVector& uid_vec,
78 const UNICHAR_ID uid) {
79 for (int i = 0; i < uid_vec.size(); ++i)
80 if (uid_vec[i] == uid) return i;
81 return -1;
82 }
83
84 // Copies UNICHAR_IDs from dst to src. Returns the number of ids copied.
85 // The function assumes that the arrays are terminated by INVALID_UNICHAR_ID
86 // and that dst has enough space for all the elements from src.
87 static inline int copy(const UNICHAR_ID src[], UNICHAR_ID dst[]) {
88 int i = 0;
89 do {
90 dst[i] = src[i];
91 } while (dst[i++] != INVALID_UNICHAR_ID);
92 return i - 1;
93 }
94
95 // Prints unichars corresponding to the unichar_ids in the given array.
96 // The function assumes that array is terminated by INVALID_UNICHAR_ID.
97 static inline void print(const UNICHAR_ID array[],
98 const UNICHARSET &unicharset) {
99 const UNICHAR_ID *ptr = array;
100 if (*ptr == INVALID_UNICHAR_ID) tprintf("[Empty]");
101 while (*ptr != INVALID_UNICHAR_ID) {
102 tprintf("%s ", unicharset.id_to_unichar(*ptr++));
103 }
104 tprintf("( ");
105 ptr = array;
106 while (*ptr != INVALID_UNICHAR_ID) tprintf("%d ", *ptr++);
107 tprintf(")\n");
108 }
109 };
110
111 // AMBIG_SPEC_LIST stores a list of dangerous ambigs that
112 // start with the same unichar (e.g. r->t rn->m rr1->m).
113 class AmbigSpec : public ELIST_LINK {
114 public:
115 AmbigSpec();
116 ~AmbigSpec() {}
117
118 // Comparator function for sorting AmbigSpec_LISTs. The lists will
119 // be sorted by their wrong_ngram arrays. Example of wrong_ngram vectors
120 // in a a sorted AmbigSpec_LIST: [9 1 3], [9 3 4], [9 8], [9, 8 1].
121 static int compare_ambig_specs(const void *spec1, const void *spec2) {
122 const AmbigSpec *s1 =
123 *reinterpret_cast<const AmbigSpec * const *>(spec1);
124 const AmbigSpec *s2 =
125 *reinterpret_cast<const AmbigSpec * const *>(spec2);
126 return UnicharIdArrayUtils::compare(s1->wrong_ngram, s2->wrong_ngram);
127 }
128
129 UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE + 1];
130 UNICHAR_ID correct_fragments[MAX_AMBIG_SIZE + 1];
131 UNICHAR_ID correct_ngram_id;
132 AmbigType type;
133 int wrong_ngram_size;
134 };
135 ELISTIZEH(AmbigSpec);
136
137 // AMBIG_TABLE[i] stores a set of ambiguities whose
138 // wrong ngram starts with unichar id i.
139 typedef GenericVector<AmbigSpec_LIST *> UnicharAmbigsVector;
140
141 class UnicharAmbigs {
142 public:
143 UnicharAmbigs() {}
144 ~UnicharAmbigs() {
145 replace_ambigs_.delete_data_pointers();
146 dang_ambigs_.delete_data_pointers();
147 one_to_one_definite_ambigs_.delete_data_pointers();
148 }
149
150 const UnicharAmbigsVector &dang_ambigs() const { return dang_ambigs_; }
151 const UnicharAmbigsVector &replace_ambigs() const { return replace_ambigs_; }
152
153 // Fills in two ambiguity tables (replaceable and dangerous) with information
154 // read from the ambigs file. An ambiguity table is an array of lists.
155 // The array is indexed by a class id. Each entry in the table provides
156 // a list of potential ambiguities which can start with the corresponding
157 // character. For example the ambiguity "rn -> m", would be located in the
158 // table at index of unicharset.unichar_to_id('r').
159 // In 1-1 ambiguities (e.g. s -> S, 1 -> I) are recorded in
160 // one_to_one_definite_ambigs_. This vector is also indexed by the class id
161 // of the wrong part of the ambiguity and each entry contains a vector of
162 // unichar ids that are ambiguous to it.
163 void LoadUnicharAmbigs(FILE *ambigs_file, inT64 end_offset, int debug_level,
164 bool use_ambigs_for_adaption, UNICHARSET *unicharset);
165
166 // Returns definite 1-1 ambigs for the given unichar id.
167 inline const UnicharIdVector *OneToOneDefiniteAmbigs(
168 UNICHAR_ID unichar_id) const {
169 if (one_to_one_definite_ambigs_.empty()) return NULL;
170 return one_to_one_definite_ambigs_[unichar_id];
171 }
172
173 // Returns a pointer to the vector with all unichar ids that appear in the
174 // 'correct' part of the ambiguity pair when the given unichar id appears
175 // in the 'wrong' part of the ambiguity. E.g. if DangAmbigs file consist of
176 // m->rn,rn->m,m->iii, UnicharAmbigsForAdaption() called with unichar id of
177 // m will return a pointer to a vector with unichar ids of r,n,i.
178 inline const UnicharIdVector *AmbigsForAdaption(
179 UNICHAR_ID unichar_id) const {
180 if (ambigs_for_adaption_.empty()) return NULL;
181 return ambigs_for_adaption_[unichar_id];
182 }
183
184 // Similar to the above, but return the vector of unichar ids for which
185 // the given unichar_id is an ambiguity (appears in the 'wrong' part of
186 // some ambiguity pair).
187 inline const UnicharIdVector *ReverseAmbigsForAdaption(
188 UNICHAR_ID unichar_id) const {
189 if (reverse_ambigs_for_adaption_.empty()) return NULL;
190 return reverse_ambigs_for_adaption_[unichar_id];
191 }
192
193 private:
194
195 bool ParseAmbiguityLine(int line_num, int version, int debug_level,
196 const UNICHARSET &unicharset, char *buffer,
197 int *TestAmbigPartSize, UNICHAR_ID *TestUnicharIds,
198 int *ReplacementAmbigPartSize,
199 char *ReplacementString, int *type);
200 void InsertIntoTable(UnicharAmbigsVector &table,
201 int TestAmbigPartSize, UNICHAR_ID *TestUnicharIds,
202 int ReplacementAmbigPartSize,
203 const char *ReplacementString, int type,
204 AmbigSpec *ambig_spec, UNICHARSET *unicharset);
205 UnicharAmbigsVector dang_ambigs_;
206 UnicharAmbigsVector replace_ambigs_;
207 GenericVector<UnicharIdVector *> one_to_one_definite_ambigs_;
208 GenericVector<UnicharIdVector *> ambigs_for_adaption_;
209 GenericVector<UnicharIdVector *> reverse_ambigs_for_adaption_;
210 };
211
212 } // namespace tesseract
213
214 #endif // TESSERACT_CCUTIL_AMBIGS_H_