1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <fcntl.h>
18 #include <sys/mman.h>
19 #include <sys/stat.h>
20 #include <sys/types.h>
21 #include <unistd.h>
22 
23 #include <algorithm>
24 
25 #include <core_jni_helpers.h>
26 #include <minikin/Hyphenator.h>
27 
28 namespace android {
29 
buildFileName(const std::string & locale)30 static std::string buildFileName(const std::string& locale) {
31     constexpr char SYSTEM_HYPHENATOR_PREFIX[] = "/system/usr/hyphen-data/hyph-";
32     constexpr char SYSTEM_HYPHENATOR_SUFFIX[] = ".hyb";
33     std::string lowerLocale;
34     lowerLocale.reserve(locale.size());
35     std::transform(locale.begin(), locale.end(), std::back_inserter(lowerLocale), ::tolower);
36     return SYSTEM_HYPHENATOR_PREFIX + lowerLocale + SYSTEM_HYPHENATOR_SUFFIX;
37 }
38 
mmapPatternFile(const std::string & locale)39 static const uint8_t* mmapPatternFile(const std::string& locale) {
40     const std::string hyFilePath = buildFileName(locale);
41     const int fd = open(hyFilePath.c_str(), O_RDONLY | O_CLOEXEC);
42     if (fd == -1) {
43         return nullptr;  // Open failed.
44     }
45 
46     struct stat st = {};
47     if (fstat(fd, &st) == -1) {  // Unlikely to happen.
48         close(fd);
49         return nullptr;
50     }
51 
52     void* ptr = mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fd, 0 /* offset */);
53     close(fd);
54     if (ptr == MAP_FAILED) {
55         return nullptr;
56     }
57     return reinterpret_cast<const uint8_t*>(ptr);
58 }
59 
addHyphenatorWithoutPatternFile(const std::string & locale,int minPrefix,int minSuffix)60 static void addHyphenatorWithoutPatternFile(const std::string& locale, int minPrefix,
61         int minSuffix) {
62     minikin::addHyphenator(locale, minikin::Hyphenator::loadBinary(
63             nullptr, minPrefix, minSuffix, locale));
64 }
65 
addHyphenator(const std::string & locale,int minPrefix,int minSuffix)66 static void addHyphenator(const std::string& locale, int minPrefix, int minSuffix) {
67     const uint8_t* ptr = mmapPatternFile(locale);
68     if (ptr == nullptr) {
69         ALOGE("Unable to find pattern file or unable to map it for %s", locale.c_str());
70         return;
71     }
72     minikin::addHyphenator(locale, minikin::Hyphenator::loadBinary(
73             ptr, minPrefix, minSuffix, locale));
74 }
75 
addHyphenatorAlias(const std::string & from,const std::string & to)76 static void addHyphenatorAlias(const std::string& from, const std::string& to) {
77     minikin::addHyphenatorAlias(from, to);
78 }
79 
init()80 static void init() {
81     // TODO: Confirm that these are the best values. Various sources suggest (1, 1), but that
82     // appears too small.
83     constexpr int INDIC_MIN_PREFIX = 2;
84     constexpr int INDIC_MIN_SUFFIX = 2;
85 
86     addHyphenator("as", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Assamese
87     addHyphenator("be", 2, 2);  // Belarusian
88     addHyphenator("bg", 2, 2);  // Bulgarian
89     addHyphenator("bn", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Bengali
90     addHyphenator("cu", 1, 2);  // Church Slavonic
91     addHyphenator("cy", 2, 3);  // Welsh
92     addHyphenator("da", 2, 2);  // Danish
93     addHyphenator("de-1901", 2, 2);  // German 1901 orthography
94     addHyphenator("de-1996", 2, 2);  // German 1996 orthography
95     addHyphenator("de-CH-1901", 2, 2);  // Swiss High German 1901 orthography
96     addHyphenator("en-GB", 2, 3);  // British English
97     addHyphenator("en-US", 2, 3);  // American English
98     addHyphenator("es", 2, 2);  // Spanish
99     addHyphenator("et", 2, 3);  // Estonian
100     addHyphenator("eu", 2, 2);  // Basque
101     addHyphenator("fr", 2, 3);  // French
102     addHyphenator("ga", 2, 3);  // Irish
103     addHyphenator("gu", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Gujarati
104     addHyphenator("hi", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Hindi
105     addHyphenator("hr", 2, 2);  // Croatian
106     addHyphenator("hu", 2, 2);  // Hungarian
107     // texhyphen sources say Armenian may be (1, 2); but that it needs confirmation.
108     // Going with a more conservative value of (2, 2) for now.
109     addHyphenator("hy", 2, 2);  // Armenian
110     addHyphenator("kn", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Kannada
111     addHyphenator("la", 2, 2);  // Latin
112     addHyphenator("ml", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Malayalam
113     addHyphenator("mn-Cyrl", 2, 2);  // Mongolian in Cyrillic script
114     addHyphenator("mr", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Marathi
115     addHyphenator("nb", 2, 2);  // Norwegian Bokmål
116     addHyphenator("nn", 2, 2);  // Norwegian Nynorsk
117     addHyphenator("or", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Oriya
118     addHyphenator("pa", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Punjabi
119     addHyphenator("pt", 2, 3);  // Portuguese
120     addHyphenator("sl", 2, 2);  // Slovenian
121     addHyphenator("ta", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Tamil
122     addHyphenator("te", INDIC_MIN_PREFIX, INDIC_MIN_SUFFIX);  // Telugu
123     addHyphenator("tk", 2, 2);  // Turkmen
124     addHyphenator("und-Ethi", 1, 1);  // Any language in Ethiopic script
125 
126     // Following two hyphenators do not have pattern files but there is some special logic based on
127     // language.
128     addHyphenatorWithoutPatternFile("ca", 2, 2);  // Catalan
129     addHyphenatorWithoutPatternFile("pl", 2, 2);  // Polish
130 
131     // English locales that fall back to en-US. The data is from CLDR. It's all English locales,
132     // minus the locales whose parent is en-001 (from supplementalData.xml, under <parentLocales>).
133     // TODO: Figure out how to get this from ICU.
134     addHyphenatorAlias("en-AS", "en-US");  // English (American Samoa)
135     addHyphenatorAlias("en-GU", "en-US");  // English (Guam)
136     addHyphenatorAlias("en-MH", "en-US");  // English (Marshall Islands)
137     addHyphenatorAlias("en-MP", "en-US");  // English (Northern Mariana Islands)
138     addHyphenatorAlias("en-PR", "en-US");  // English (Puerto Rico)
139     addHyphenatorAlias("en-UM", "en-US");  // English (United States Minor Outlying Islands)
140     addHyphenatorAlias("en-VI", "en-US");  // English (Virgin Islands)
141 
142     // All English locales other than those falling back to en-US are mapped to en-GB.
143     addHyphenatorAlias("en", "en-GB");
144 
145     // For German, we're assuming the 1996 (and later) orthography by default.
146     addHyphenatorAlias("de", "de-1996");
147     // Liechtenstein uses the Swiss hyphenation rules for the 1901 orthography.
148     addHyphenatorAlias("de-LI-1901", "de-CH-1901");
149 
150     // Norwegian is very probably Norwegian Bokmål.
151     addHyphenatorAlias("no", "nb");
152 
153     // Use mn-Cyrl. According to CLDR's likelySubtags.xml, mn is most likely to be mn-Cyrl.
154     addHyphenatorAlias("mn", "mn-Cyrl");  // Mongolian
155 
156     // Fall back to Ethiopic script for languages likely to be written in Ethiopic.
157     // Data is from CLDR's likelySubtags.xml.
158     // TODO: Convert this to a mechanism using ICU4J's ULocale#addLikelySubtags().
159     addHyphenatorAlias("am", "und-Ethi");  // Amharic
160     addHyphenatorAlias("byn", "und-Ethi");  // Blin
161     addHyphenatorAlias("gez", "und-Ethi");  // Geʻez
162     addHyphenatorAlias("ti", "und-Ethi");  // Tigrinya
163     addHyphenatorAlias("wal", "und-Ethi");  // Wolaytta
164 
165     // Use Hindi as a fallback hyphenator for all languages written in Devanagari, etc. This makes
166     // sense because our Indic patterns are not really linguistic, but script-based.
167     addHyphenatorAlias("und-Beng", "bn");  // Bengali
168     addHyphenatorAlias("und-Deva", "hi");  // Devanagari -> Hindi
169     addHyphenatorAlias("und-Gujr", "gu");  // Gujarati
170     addHyphenatorAlias("und-Guru", "pa");  // Gurmukhi -> Punjabi
171     addHyphenatorAlias("und-Knda", "kn");  // Kannada
172     addHyphenatorAlias("und-Mlym", "ml");  // Malayalam
173     addHyphenatorAlias("und-Orya", "or");  // Oriya
174     addHyphenatorAlias("und-Taml", "ta");  // Tamil
175     addHyphenatorAlias("und-Telu", "te");  // Telugu
176 }
177 
178 static const JNINativeMethod gMethods[] = {
179     {"nInit", "()V", (void*) init},
180 };
181 
register_android_text_Hyphenator(JNIEnv * env)182 int register_android_text_Hyphenator(JNIEnv* env) {
183     return RegisterMethodsOrDie(env, "android/text/Hyphenator", gMethods, NELEM(gMethods));
184 }
185 
186 }  // namespace android
187