Tesseract 3.02

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
unicharset_extractor.cpp
Go to the documentation of this file.
1
2 // File: unicharset_extractor.cpp
3 // Description: Unicode character/ligature set extractor.
4 // Author: Thomas Kielbus
5 // Created: Wed Jun 28 17:05:01 PDT 2006
6 //
7 // (C) Copyright 2006, Google Inc.
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 // http://www.apache.org/licenses/LICENSE-2.0
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
17 //
19
20 // Given a list of box files on the command line, this program generates a file
21 // containing a unicharset, a list of all the characters used by Tesseract
22 //
23 // The file contains the size of the set on the first line, and then one
24 // unichar per line.
25
26 #include <stdio.h>
27 /*
28 ** Include automatically generated configuration file if running autoconf
29 */
30 #ifdef HAVE_CONFIG_H
31 #include "config_auto.h"
32 #endif
33 #if defined(HAVE_WCHAR_T) || defined(_WIN32) || defined(GOOGLE3)
34 #include <wchar.h>
35 #include <wctype.h>
36 #define USING_WCTYPE
37 #endif
38 #include <locale.h>
39
40 #include "boxread.h"
41 #include "rect.h"
42 #include "strngs.h"
43 #include "tessopt.h"
44 #include "unichar.h"
45 #include "unicharset.h"
46
47 static const char* const kUnicharsetFileName = "unicharset";
48
49 UNICHAR_ID wc_to_unichar_id(const UNICHARSET &unicharset, int wc) {
50 UNICHAR uch(wc);
51 char *unichar = uch.utf8_str();
52 UNICHAR_ID unichar_id = unicharset.unichar_to_id(unichar);
53 delete[] unichar;
54 return unichar_id;
55 }
56
57 // Set character properties using wctype if we have it.
58 // Contributed by piggy@gmail.com.
59 // Modified by Ray to use UNICHAR for unicode conversion
60 // and to check for wctype using autoconf/presence of windows.
61 void set_properties(UNICHARSET *unicharset, const char* const c_string) {
62 #ifdef USING_WCTYPE
63 UNICHAR_ID id;
64 int wc;
65
66 // Convert the string to a unichar id.
67 id = unicharset->unichar_to_id(c_string);
68
69 // Set the other_case property to be this unichar id by default.
70 unicharset->set_other_case(id, id);
71
72 int step = UNICHAR::utf8_step(c_string);
73 if (step == 0)
74 return; // Invalid utf-8.
75
76 // Get the next Unicode code point in the string.
77 UNICHAR ch(c_string, step);
78 wc = ch.first_uni();
79
80 /* Copy the properties. */
81 if (iswalpha(wc)) {
82 unicharset->set_isalpha(id, 1);
83 if (iswlower(wc)) {
84 unicharset->set_islower(id, 1);
85 unicharset->set_other_case(id, wc_to_unichar_id(*unicharset,
86 towupper(wc)));
87 }
88 if (iswupper(wc)) {
89 unicharset->set_isupper(id, 1);
90 unicharset->set_other_case(id, wc_to_unichar_id(*unicharset,
91 towlower(wc)));
92 }
93 }
94 if (iswdigit(wc))
95 unicharset->set_isdigit(id, 1);
96 if(iswpunct(wc))
97 unicharset->set_ispunctuation(id, 1);
98
99 #endif
100 }
101
102 int main(int argc, char** argv) {
103 int option;
104 const char* output_directory = ".";
105 STRING unicharset_file_name;
106 UNICHARSET unicharset;
107
108 setlocale(LC_ALL, "");
109 // Space character needed to represent NIL classification
110 unicharset.unichar_insert(" ");
111
112 // Print usage
113 if (argc <= 1) {
114 printf("Usage: %s [-D DIRECTORY] FILE...\n", argv[0]);
115 exit(1);
116
117 }
118
119 // Parse arguments
120 while ((option = tessopt(argc, argv, "D" )) != EOF) {
121 switch (option) {
122 case 'D':
123 output_directory = tessoptarg;
124 ++tessoptind;
125 break;
126 }
127 }
128
129 // Save file name
130 unicharset_file_name = output_directory;
131 unicharset_file_name += "/";
132 unicharset_file_name += kUnicharsetFileName;
133
134 // Load box files
135 for (; tessoptind < argc; ++tessoptind) {
136 printf("Extracting unicharset from %s\n", argv[tessoptind]);
137
138 FILE* box_file = fopen(argv[tessoptind], "rb");
139 if (box_file == NULL) {
140 printf("Cannot open box file %s\n", argv[tessoptind]);
141 return -1;
142 }
143
144 TBOX box;
145 STRING unichar_string;
146 int line_number = 0;
147 while (ReadNextBox(&line_number, box_file, &unichar_string, &box)) {
148 unicharset.unichar_insert(unichar_string.string());
149 set_properties(&unicharset, unichar_string.string());
150 }
151 }
152
153 // Write unicharset file
154 if (unicharset.save_to_file(unicharset_file_name.string())) {
155 printf("Wrote unicharset file %s.\n", unicharset_file_name.string());
156 }
157 else {
158 printf("Cannot save unicharset file %s.\n", unicharset_file_name.string());
159 return -1;
160 }
161 return 0;
162 }