1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "dictionary/structure/v4/content/language_model_dict_content.h"
18
19 #include <gtest/gtest.h>
20
21 #include <array>
22 #include <unordered_set>
23
24 #include "utils/int_array_view.h"
25
26 namespace latinime {
27 namespace {
28
TEST(LanguageModelDictContentTest,TestUnigramProbability)29 TEST(LanguageModelDictContentTest, TestUnigramProbability) {
30 LanguageModelDictContent languageModelDictContent(false /* useHistoricalInfo */);
31
32 const int flag = 0xF0;
33 const int probability = 10;
34 const int wordId = 100;
35 const ProbabilityEntry probabilityEntry(flag, probability);
36 languageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry);
37 const ProbabilityEntry entry =
38 languageModelDictContent.getProbabilityEntry(wordId);
39 EXPECT_EQ(flag, entry.getFlags());
40 EXPECT_EQ(probability, entry.getProbability());
41
42 // Remove
43 EXPECT_TRUE(languageModelDictContent.removeProbabilityEntry(wordId));
44 EXPECT_FALSE(languageModelDictContent.getProbabilityEntry(wordId).isValid());
45 EXPECT_FALSE(languageModelDictContent.removeProbabilityEntry(wordId));
46 EXPECT_TRUE(languageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));
47 EXPECT_TRUE(languageModelDictContent.getProbabilityEntry(wordId).isValid());
48 }
49
TEST(LanguageModelDictContentTest,TestUnigramProbabilityWithHistoricalInfo)50 TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) {
51 LanguageModelDictContent languageModelDictContent(true /* useHistoricalInfo */);
52
53 const int flag = 0xF0;
54 const int timestamp = 0x3FFFFFFF;
55 const int count = 10;
56 const int wordId = 100;
57 const HistoricalInfo historicalInfo(timestamp, 0 /* level */, count);
58 const ProbabilityEntry probabilityEntry(flag, &historicalInfo);
59 languageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry);
60 const ProbabilityEntry entry = languageModelDictContent.getProbabilityEntry(wordId);
61 EXPECT_EQ(flag, entry.getFlags());
62 EXPECT_EQ(timestamp, entry.getHistoricalInfo()->getTimestamp());
63 EXPECT_EQ(count, entry.getHistoricalInfo()->getCount());
64
65 // Remove
66 EXPECT_TRUE(languageModelDictContent.removeProbabilityEntry(wordId));
67 EXPECT_FALSE(languageModelDictContent.getProbabilityEntry(wordId).isValid());
68 EXPECT_FALSE(languageModelDictContent.removeProbabilityEntry(wordId));
69 EXPECT_TRUE(languageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));
70 EXPECT_TRUE(languageModelDictContent.removeProbabilityEntry(wordId));
71 }
72
TEST(LanguageModelDictContentTest,TestIterateProbabilityEntry)73 TEST(LanguageModelDictContentTest, TestIterateProbabilityEntry) {
74 LanguageModelDictContent languageModelDictContent(false /* useHistoricalInfo */);
75
76 const ProbabilityEntry originalEntry(0xFC, 100);
77
78 const int wordIds[] = { 1, 2, 3, 4, 5 };
79 for (const int wordId : wordIds) {
80 languageModelDictContent.setProbabilityEntry(wordId, &originalEntry);
81 }
82 std::unordered_set<int> wordIdSet(std::begin(wordIds), std::end(wordIds));
83 for (const auto& entry : languageModelDictContent.getProbabilityEntries(WordIdArrayView())) {
84 EXPECT_EQ(originalEntry.getFlags(), entry.getProbabilityEntry().getFlags());
85 EXPECT_EQ(originalEntry.getProbability(), entry.getProbabilityEntry().getProbability());
86 wordIdSet.erase(entry.getWordId());
87 }
88 EXPECT_TRUE(wordIdSet.empty());
89 }
90
TEST(LanguageModelDictContentTest,TestGetWordProbability)91 TEST(LanguageModelDictContentTest, TestGetWordProbability) {
92 LanguageModelDictContent languageModelDictContent(false /* useHistoricalInfo */);
93
94 const int flag = 0xFF;
95 const int probability = 10;
96 const int bigramProbability = 20;
97 const int trigramProbability = 30;
98 const int wordId = 100;
99 const std::array<int, 2> prevWordIdArray = {{ 1, 2 }};
100 const WordIdArrayView prevWordIds = WordIdArrayView::fromArray(prevWordIdArray);
101
102 const ProbabilityEntry probabilityEntry(flag, probability);
103 languageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry);
104 const ProbabilityEntry bigramProbabilityEntry(flag, bigramProbability);
105 languageModelDictContent.setProbabilityEntry(prevWordIds[0], &probabilityEntry);
106 languageModelDictContent.setNgramProbabilityEntry(prevWordIds.limit(1), wordId,
107 &bigramProbabilityEntry);
108 EXPECT_EQ(bigramProbability, languageModelDictContent.getWordAttributes(prevWordIds, wordId,
109 false /* mustMatchAllPrevWords */, nullptr /* headerPolicy */).getProbability());
110 const ProbabilityEntry trigramProbabilityEntry(flag, trigramProbability);
111 languageModelDictContent.setNgramProbabilityEntry(prevWordIds.limit(1),
112 prevWordIds[1], &probabilityEntry);
113 languageModelDictContent.setNgramProbabilityEntry(prevWordIds.limit(2), wordId,
114 &trigramProbabilityEntry);
115 EXPECT_EQ(trigramProbability, languageModelDictContent.getWordAttributes(prevWordIds, wordId,
116 false /* mustMatchAllPrevWords */, nullptr /* headerPolicy */).getProbability());
117 }
118
119 } // namespace
120 } // namespace latinime
121