1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <fcntl.h>
18 #include <sys/mman.h>
19 #include <sys/stat.h>
20 #include <sys/types.h>
21 #include <unistd.h>
22 
23 #include <algorithm>
24 
25 #include <core_jni_helpers.h>
26 #include <minikin/Hyphenator.h>
27 
28 namespace android {
29 
buildFileName(const std::string & locale)30 static std::string buildFileName(const std::string& locale) {
31     constexpr char SYSTEM_HYPHENATOR_PREFIX[] = "/system/usr/hyphen-data/hyph-";
32     constexpr char SYSTEM_HYPHENATOR_SUFFIX[] = ".hyb";
33     std::string lowerLocale;
34     lowerLocale.reserve(locale.size());
35     std::transform(locale.begin(), locale.end(), std::back_inserter(lowerLocale), ::tolower);
36     return SYSTEM_HYPHENATOR_PREFIX + lowerLocale + SYSTEM_HYPHENATOR_SUFFIX;
37 }
38 
mmapPatternFile(const std::string & locale)39 static const uint8_t* mmapPatternFile(const std::string& locale) {
40     const std::string hyFilePath = buildFileName(locale);
41     const int fd = open(hyFilePath.c_str(), O_RDONLY | O_CLOEXEC);
42     if (fd == -1) {
43         return nullptr;  // Open failed.
44     }
45 
46     struct stat st = {};
47     if (fstat(fd, &st) == -1) {  // Unlikely to happen.
48         close(fd);
49         return nullptr;
50     }
51 
52     void* ptr = mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fd, 0 /* offset */);
53     close(fd);
54     if (ptr == MAP_FAILED) {
55         return nullptr;
56     }
57     return reinterpret_cast<const uint8_t*>(ptr);
58 }
59 
addHyphenatorWithoutPatternFile(const std::string & locale,int minPrefix,int minSuffix)60 static void addHyphenatorWithoutPatternFile(const std::string& locale, int minPrefix,
61         int minSuffix) {
62     minikin::addHyphenator(locale, minikin::Hyphenator::loadBinary(
63             nullptr, minPrefix, minSuffix, locale));
64 }
65 
addHyphenator(const std::string & locale,int minPrefix,int minSuffix)66 static void addHyphenator(const std::string& locale, int minPrefix, int minSuffix) {
67     const uint8_t* ptr = mmapPatternFile(locale);
68     if (ptr == nullptr) {
69         ALOGE("Unable to find pattern file or unable to map it for %s", locale.c_str());
70         return;
71     }
72     minikin::addHyphenator(locale, minikin::Hyphenator::loadBinary(
73             ptr, minPrefix, minSuffix, locale));
74 }
75 
addHyphenatorAlias(const std::string & from,const std::string & to)76 static void addHyphenatorAlias(const std::string& from, const std::string& to) {
77     minikin::addHyphenatorAlias(from, to);
78 }
79 
init()80 static void init() {
81     // TODO: Confirm that these are the best values. Various sources suggest (1, 1), but that
82     // appears too small.
83     constexpr int INDIC_MIN_PREFIX = 2;
84     constexpr int INDIC_MIN_SUFFIX = 2;
85 
86     addHyphenator("af", 1, 1);  // Afrikaans
87     addHyphenator("am", 1, 1);  // Amharic
88     addHyphenator("as", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Assamese
89     addHyphenator("be", 2, 2);  // Belarusian
90     addHyphenator("bg", 2, 2);  // Bulgarian
91     addHyphenator("bn", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Bengali
92     addHyphenator("cs", 2, 2);  // Czech
93     addHyphenator("cu", 1, 2);  // Church Slavonic
94     addHyphenator("cy", 2, 3);  // Welsh
95     addHyphenator("da", 2, 2);  // Danish
96     addHyphenator("de-1901", 2, 2);  // German 1901 orthography
97     addHyphenator("de-1996", 2, 2);  // German 1996 orthography
98     addHyphenator("de-CH-1901", 2, 2);  // Swiss High German 1901 orthography
99     addHyphenator("el", 1, 1);  // Greek
100     addHyphenator("en-GB", 2, 3);  // British English
101     addHyphenator("en-US", 2, 3);  // American English
102     addHyphenator("es", 2, 2);  // Spanish
103     addHyphenator("et", 2, 3);  // Estonian
104     addHyphenator("eu", 2, 2);  // Basque
105     addHyphenator("fr", 2, 3);  // French
106     addHyphenator("ga", 2, 3);  // Irish
107     addHyphenator("gl", 2, 2);  // Galician
108     addHyphenator("gu", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Gujarati
109     addHyphenator("hi", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Hindi
110     addHyphenator("hr", 2, 2);  // Croatian
111     addHyphenator("hu", 2, 2);  // Hungarian
112     // texhyphen sources say Armenian may be (1, 2); but that it needs confirmation.
113     // Going with a more conservative value of (2, 2) for now.
114     addHyphenator("hy", 2, 2);  // Armenian
115     addHyphenator("it", 2, 2);  // Italian
116     addHyphenator("ka", 1, 2);  // Georgian
117     addHyphenator("kn", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Kannada
118     addHyphenator("la", 2, 2);  // Latin
119     addHyphenator("lt", 2, 2);  // Lithuanian
120     addHyphenator("lv", 2, 2);  // Latvian
121     addHyphenator("ml", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Malayalam
122     addHyphenator("mn-Cyrl", 2, 2);  // Mongolian in Cyrillic script
123     addHyphenator("mr", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Marathi
124     addHyphenator("nb", 2, 2);  // Norwegian Bokmål
125     addHyphenator("nl", 2, 2);  // Dutch
126     addHyphenator("nn", 2, 2);  // Norwegian Nynorsk
127     addHyphenator("or", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Oriya
128     addHyphenator("pa", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Punjabi
129     addHyphenator("pl", 2, 2);  // Polish
130     addHyphenator("pt", 2, 3);  // Portuguese
131     addHyphenator("ru", 2, 2);  // Russian
132     addHyphenator("sk", 2, 2);  // Slovak
133     addHyphenator("sl", 2, 2);  // Slovenian
134     addHyphenator("sq", 2, 2);  // Albanian
135     addHyphenator("sv", 1, 2);  // Swedish
136     addHyphenator("ta", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Tamil
137     addHyphenator("te", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Telugu
138     addHyphenator("tk", 2, 2);  // Turkmen
139     addHyphenator("uk", 2, 2);  // Ukrainian
140     addHyphenator("und-Ethi", 1, 1);  // Any language in Ethiopic script
141 
142     // Following two hyphenators do not have pattern files but there is some special logic based on
143     // language.
144     addHyphenatorWithoutPatternFile("ca", 2, 2);  // Catalan
145 
146     // English locales that fall back to en-US. The data is from CLDR. It's all English locales,
147     // minus the locales whose parent is en-001 (from supplementalData.xml, under <parentLocales>).
148     // TODO: Figure out how to get this from ICU.
149     addHyphenatorAlias("en-AS", "en-US");  // English (American Samoa)
150     addHyphenatorAlias("en-GU", "en-US");  // English (Guam)
151     addHyphenatorAlias("en-MH", "en-US");  // English (Marshall Islands)
152     addHyphenatorAlias("en-MP", "en-US");  // English (Northern Mariana Islands)
153     addHyphenatorAlias("en-PR", "en-US");  // English (Puerto Rico)
154     addHyphenatorAlias("en-UM", "en-US");  // English (United States Minor Outlying Islands)
155     addHyphenatorAlias("en-VI", "en-US");  // English (Virgin Islands)
156 
157     // All English locales other than those falling back to en-US are mapped to en-GB.
158     addHyphenatorAlias("en", "en-GB");
159 
160     // For German, we're assuming the 1996 (and later) orthography by default.
161     addHyphenatorAlias("de", "de-1996");
162     // Liechtenstein uses the Swiss hyphenation rules for the 1901 orthography.
163     addHyphenatorAlias("de-LI-1901", "de-CH-1901");
164 
165     // Norwegian is very probably Norwegian Bokmål.
166     addHyphenatorAlias("no", "nb");
167 
168     // Use mn-Cyrl. According to CLDR's likelySubtags.xml, mn is most likely to be mn-Cyrl.
169     addHyphenatorAlias("mn", "mn-Cyrl");  // Mongolian
170 
171     // Fall back to Ethiopic script for languages likely to be written in Ethiopic.
172     // Data is from CLDR's likelySubtags.xml.
173     // TODO: Convert this to a mechanism using ICU4J's ULocale#addLikelySubtags().
174     addHyphenatorAlias("am", "und-Ethi");  // Amharic
175     addHyphenatorAlias("byn", "und-Ethi");  // Blin
176     addHyphenatorAlias("gez", "und-Ethi");  // Geʻez
177     addHyphenatorAlias("ti", "und-Ethi");  // Tigrinya
178     addHyphenatorAlias("wal", "und-Ethi");  // Wolaytta
179 
180     // Use Hindi as a fallback hyphenator for all languages written in Devanagari, etc. This makes
181     // sense because our Indic patterns are not really linguistic, but script-based.
182     addHyphenatorAlias("und-Beng", "bn");  // Bengali
183     addHyphenatorAlias("und-Deva", "hi");  // Devanagari -> Hindi
184     addHyphenatorAlias("und-Gujr", "gu");  // Gujarati
185     addHyphenatorAlias("und-Guru", "pa");  // Gurmukhi -> Punjabi
186     addHyphenatorAlias("und-Knda", "kn");  // Kannada
187     addHyphenatorAlias("und-Mlym", "ml");  // Malayalam
188     addHyphenatorAlias("und-Orya", "or");  // Oriya
189     addHyphenatorAlias("und-Taml", "ta");  // Tamil
190     addHyphenatorAlias("und-Telu", "te");  // Telugu
191 }
192 
193 static const JNINativeMethod gMethods[] = {
194     {"nInit", "()V", (void*) init},
195 };
196 
register_android_text_Hyphenator(JNIEnv * env)197 int register_android_text_Hyphenator(JNIEnv* env) {
198     return RegisterMethodsOrDie(env, "android/text/Hyphenator", gMethods, NELEM(gMethods));
199 }
200 
201 }  // namespace android
202