1 /*
2  * Copyright (C) 2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define MLOG_TAG "Scanner"
17 
18 #include "media_scanner.h"
19 
20 #include "directory_ex.h"
21 #include "hitrace_meter.h"
22 #include "ipc_skeleton.h"
23 #include "rdb_predicates.h"
24 #include "media_file_utils.h"
25 #include "media_file_uri.h"
26 #include "media_log.h"
27 #include "medialibrary_unistore_manager.h"
28 #include "medialibrary_errno.h"
29 #include "medialibrary_rdb_utils.h"
30 #include "medialibrary_notify.h"
31 #include "mimetype_utils.h"
32 #include "post_event_utils.h"
33 #include "photo_map_column.h"
34 #include "photo_album_column.h"
35 #include "vision_column.h"
36 #include "moving_photo_file_utils.h"
37 
38 namespace OHOS {
39 namespace Media {
40 using namespace std;
41 using namespace OHOS::AppExecFwk;
42 using namespace OHOS::DataShare;
43 
MediaScannerObj(const std::string & path,const std::shared_ptr<IMediaScannerCallback> & callback,MediaScannerObj::ScanType type,MediaLibraryApi api)44 MediaScannerObj::MediaScannerObj(const std::string &path, const std::shared_ptr<IMediaScannerCallback> &callback,
45     MediaScannerObj::ScanType type, MediaLibraryApi api) : type_(type), callback_(callback), api_(api)
46 {
47     if (type_ == DIRECTORY) {
48         dir_ = path;
49     } else if (type_ == FILE) {
50         path_ = path;
51     }
52     // when path is /Photo, it means update or clone scene
53     skipPhoto_ = path.compare("/storage/cloud/files/Photo") != 0;
54     stopFlag_ = make_shared<bool>(false);
55 }
56 
MediaScannerObj(MediaScannerObj::ScanType type,MediaLibraryApi api)57 MediaScannerObj::MediaScannerObj(MediaScannerObj::ScanType type, MediaLibraryApi api) : type_(type), api_(api)
58 {
59 }
60 
SetStopFlag(std::shared_ptr<bool> & flag)61 void MediaScannerObj::SetStopFlag(std::shared_ptr<bool> &flag)
62 {
63     stopFlag_ = flag;
64 }
65 
SetErrorPath(const std::string & path)66 void MediaScannerObj::SetErrorPath(const std::string &path)
67 {
68     errorPath_ = path;
69 }
70 
SetForceScan(bool isForceScan)71 void MediaScannerObj::SetForceScan(bool isForceScan)
72 {
73     isForceScan_ = isForceScan;
74 }
75 
SetFileId(int32_t fileId)76 void MediaScannerObj::SetFileId(int32_t fileId)
77 {
78     fileId_ = fileId;
79 }
80 
SetIsSkipAlbumUpdate(bool isSkipAlbumUpdate)81 void MediaScannerObj::SetIsSkipAlbumUpdate(bool isSkipAlbumUpdate)
82 {
83     isSkipAlbumUpdate_ = isSkipAlbumUpdate;
84 }
85 
ScanFile()86 int32_t MediaScannerObj::ScanFile()
87 {
88     MEDIA_DEBUG_LOG("scan file %{private}s", path_.c_str());
89 
90     int32_t ret = ScanFileInternal();
91     if (ret != E_OK) {
92         MEDIA_ERR_LOG("ScanFileInternal err %{public}d", ret);
93         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, ret},
94             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
95         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
96     }
97 
98     (void)InvokeCallback(ret);
99 
100     return ret;
101 }
102 
ScanDir()103 int32_t MediaScannerObj::ScanDir()
104 {
105     MEDIA_INFO_LOG("scan dir %{public}s", dir_.c_str());
106     int64_t start = MediaFileUtils::UTCTimeMilliSeconds();
107     int32_t ret = ScanDirInternal();
108     int64_t end = MediaFileUtils::UTCTimeMilliSeconds();
109     MEDIA_INFO_LOG("scan dir cost: %{public}ld", (long)(end - start));
110     if (ret != E_OK) {
111         MEDIA_ERR_LOG("ScanDirInternal err %{public}d", ret);
112         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, ret},
113             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
114         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
115     }
116 
117     (void)InvokeCallback(ret);
118 
119     return ret;
120 }
121 
Scan()122 void MediaScannerObj::Scan()
123 {
124     switch (type_) {
125         case FILE:
126             ScanFile();
127             break;
128         case DIRECTORY:
129             ScanDir();
130             break;
131         case START:
132             Start();
133             break;
134         case ERROR:
135             ScanError();
136             break;
137         case SET_ERROR:
138             SetError();
139             break;
140         default:
141             break;
142     }
143 }
144 
InvokeCallback(int32_t err)145 int32_t MediaScannerObj::InvokeCallback(int32_t err)
146 {
147     if (callback_ == nullptr) {
148         return E_OK;
149     }
150 
151     return callback_->OnScanFinished(err, uri_, path_);
152 }
153 
CommitTransaction()154 int32_t MediaScannerObj::CommitTransaction()
155 {
156     unordered_set<MediaType> mediaTypeSet = {};
157     string uri;
158     unique_ptr<Metadata> data;
159     string tableName;
160 
161     // will begin a transaction in later pr
162     for (uint32_t i = 0; i < dataBuffer_.size(); i++) {
163         data = move(dataBuffer_[i]);
164         if (data->GetFileId() != FILE_ID_DEFAULT) {
165             MediaLibraryApi api = skipPhoto_ ? MediaLibraryApi::API_OLD : MediaLibraryApi::API_10;
166             uri = mediaScannerDb_->UpdateMetadata(*data, tableName, api, skipPhoto_);
167             scannedIds_.insert(make_pair(tableName, data->GetFileId()));
168         } else {
169             uri = mediaScannerDb_->InsertMetadata(*data, tableName);
170             scannedIds_.insert(make_pair(tableName,
171                 stoi(MediaFileUtils::GetIdFromUri(uri))));
172         }
173 
174         // set uri for callback
175         uri_ = uri;
176         mediaTypeSet.insert(data->GetFileMediaType());
177     }
178 
179     dataBuffer_.clear();
180 
181     mediaScannerDb_->UpdateAlbumInfo();
182     for (const MediaType &mediaType : mediaTypeSet) {
183         mediaScannerDb_->NotifyDatabaseChange(mediaType);
184     }
185 
186     return E_OK;
187 }
188 
AddToTransaction()189 int32_t MediaScannerObj::AddToTransaction()
190 {
191     dataBuffer_.emplace_back(move(data_));
192     if (dataBuffer_.size() >= MAX_BATCH_SIZE) {
193         return CommitTransaction();
194     }
195 
196     return E_OK;
197 }
198 
GetUriWithoutSeg(const string & oldUri)199 string GetUriWithoutSeg(const string &oldUri)
200 {
201     size_t questionMaskPoint = oldUri.rfind('?');
202     if (questionMaskPoint != string::npos) {
203         return oldUri.substr(0, questionMaskPoint);
204     }
205     return oldUri;
206 }
207 
GetAlbumId(const shared_ptr<MediaLibraryRdbStore> rdbStore,const string & albumName)208 static string GetAlbumId(const shared_ptr<MediaLibraryRdbStore> rdbStore,
209     const string &albumName)
210 {
211     NativeRdb::RdbPredicates predicates(ANALYSIS_ALBUM_TABLE);
212     predicates.EqualTo(PhotoAlbumColumns::ALBUM_NAME, albumName);
213     if (rdbStore == nullptr) {
214         MEDIA_ERR_LOG("rdbStore nullptr");
215         return "";
216     }
217     vector<string> columns = {PhotoAlbumColumns::ALBUM_ID};
218     auto albumResult = rdbStore->Query(predicates, columns);
219     if (albumResult == nullptr || albumResult->GoToNextRow() != NativeRdb::E_OK) {
220         MEDIA_ERR_LOG("query fails, no target shootingmode");
221         return "";
222     }
223     int32_t index = 0;
224     if (albumResult->GetColumnIndex(PhotoAlbumColumns::ALBUM_ID, index) != NativeRdb::E_OK) {
225         MEDIA_ERR_LOG("get column index fails");
226         return "";
227     }
228     string album_id = "";
229     if (albumResult->GetString(index, album_id) != NativeRdb::E_OK) {
230         return "";
231     }
232     return album_id;
233 }
234 
MaintainShootingModeMap(std::unique_ptr<Metadata> & data,const shared_ptr<MediaLibraryRdbStore> rdbStore)235 static int32_t MaintainShootingModeMap(std::unique_ptr<Metadata> &data,
236     const shared_ptr<MediaLibraryRdbStore> rdbStore)
237 {
238     string insertShootingModeSql = "INSERT OR IGNORE INTO " + ANALYSIS_PHOTO_MAP_TABLE +
239         "(" + PhotoMap::ALBUM_ID + "," + PhotoMap::ASSET_ID + ") SELECT AnalysisAlbum.album_id, " +
240         "Photos.file_id FROM " + PhotoColumn::PHOTOS_TABLE + " JOIN " +
241         ANALYSIS_ALBUM_TABLE + " ON Photos.shooting_mode=AnalysisAlbum.album_name WHERE file_id = " +
242         to_string(data->GetFileId()) + " AND AnalysisAlbum.album_subtype = " +
243         to_string(PhotoAlbumSubType::SHOOTING_MODE);
244     int32_t result = rdbStore->ExecuteSql(insertShootingModeSql);
245     return result;
246 }
247 
MaintainAlbumRelationship(std::unique_ptr<Metadata> & data)248 static int32_t MaintainAlbumRelationship(std::unique_ptr<Metadata> &data)
249 {
250     auto rdbStore = MediaLibraryUnistoreManager::GetInstance().GetRdbStore();
251     if (rdbStore == nullptr) {
252         MEDIA_ERR_LOG("rdbstore is nullptr");
253         return E_HAS_DB_ERROR;
254     }
255     auto ret = MaintainShootingModeMap(data, rdbStore);
256     if (ret != E_OK) {
257         return ret;
258     }
259     string album_id = GetAlbumId(rdbStore, data->GetShootingMode());
260     if (album_id.empty()) {
261         MEDIA_ERR_LOG("Failed to query album_id,Album cover and count update fails");
262         return E_ERR;
263     }
264     MediaLibraryRdbUtils::UpdateAnalysisAlbumInternal(rdbStore, {album_id});
265     return E_OK;
266 }
267 
Commit()268 int32_t MediaScannerObj::Commit()
269 {
270     string tableName;
271     auto watch = MediaLibraryNotify::GetInstance();
272     if (data_->GetFileId() != FILE_ID_DEFAULT) {
273         uri_ = mediaScannerDb_->UpdateMetadata(*data_, tableName, api_);
274         if (!isSkipAlbumUpdate_) {
275             mediaScannerDb_->UpdateAlbumInfoByMetaData(*data_);
276         }
277         if (watch != nullptr && data_->GetIsTemp() == FILE_IS_TEMP_DEFAULT) {
278             if (data_->GetForAdd()) {
279                 watch->Notify(GetUriWithoutSeg(uri_), NOTIFY_ADD);
280             } else {
281                 watch->Notify(GetUriWithoutSeg(uri_), NOTIFY_UPDATE);
282             }
283         }
284     } else {
285         MEDIA_INFO_LOG("insert new file: %{public}s", data_->GetFilePath().c_str());
286         uri_ = mediaScannerDb_->InsertMetadata(*data_, tableName, api_);
287         mediaScannerDb_->UpdateAlbumInfoByMetaData(*data_);
288         if (watch != nullptr && data_->GetIsTemp() == FILE_IS_TEMP_DEFAULT) {
289             watch->Notify(GetUriWithoutSeg(uri_), NOTIFY_ADD);
290         }
291     }
292 
293     if (!data_->GetShootingMode().empty() && !isSkipAlbumUpdate_) {
294         auto err = MaintainAlbumRelationship(data_);
295         if (err != E_OK) {
296             return err;
297         }
298     }
299 
300     // notify change
301     mediaScannerDb_->NotifyDatabaseChange(data_->GetFileMediaType());
302     data_ = nullptr;
303 
304     return E_OK;
305 }
306 
GetMediaInfo()307 int32_t MediaScannerObj::GetMediaInfo()
308 {
309     auto pos = data_->GetFileMimeType().find_first_of("/");
310     string mimePrefix = data_->GetFileMimeType().substr(0, pos) + "/*";
311     if (find(EXTRACTOR_SUPPORTED_MIME.begin(), EXTRACTOR_SUPPORTED_MIME.end(),
312         mimePrefix) != EXTRACTOR_SUPPORTED_MIME.end()) {
313         return MetadataExtractor::Extract(data_);
314     }
315 
316     return E_OK;
317 }
318 
319 #ifdef MEDIALIBRARY_COMPATIBILITY
SetPhotoSubType(const string & parent)320 void MediaScannerObj::SetPhotoSubType(const string &parent)
321 {
322     if ((data_->GetFileMediaType() != MediaType::MEDIA_TYPE_IMAGE) &&
323         (data_->GetFileMediaType() != MediaType::MEDIA_TYPE_VIDEO)) {
324         return;
325     }
326 
327     string parentPath = parent + SLASH_CHAR;
328     if (parentPath.find(ROOT_MEDIA_DIR) != 0) {
329         return;
330     }
331 
332     size_t len = ROOT_MEDIA_DIR.length();
333     parentPath.erase(0, len);
334 
335     if (parentPath == CAMERA_PATH) {
336         data_->SetPhotoSubType(static_cast<int32_t>(PhotoSubType::CAMERA));
337     } else if ((parentPath == SCREEN_RECORD_PATH) || (parentPath == SCREEN_SHOT_PATH)) {
338         data_->SetPhotoSubType(static_cast<int32_t>(PhotoSubType::SCREENSHOT));
339     }
340 }
341 #endif
342 
SkipBucket(const string & albumPath)343 static bool SkipBucket(const string &albumPath)
344 {
345     string path = albumPath;
346     if (path.back() != SLASH_CHAR) {
347         path += SLASH_CHAR;
348     }
349 
350     if (path.find(ROOT_MEDIA_DIR + AUDIO_BUCKET + SLASH_CHAR) != string::npos) {
351         return true;
352     }
353 
354     if (path.find(ROOT_MEDIA_DIR + PHOTO_BUCKET + SLASH_CHAR) != string::npos) {
355         return true;
356     }
357     return false;
358 }
359 
GetParentDirInfo(const string & parent,int32_t parentId)360 int32_t MediaScannerObj::GetParentDirInfo(const string &parent, int32_t parentId)
361 {
362     if (api_ == MediaLibraryApi::API_10 || SkipBucket(parent)) {
363         return E_OK;
364     }
365     size_t len = ROOT_MEDIA_DIR.length();
366     string parentPath = parent + SLASH_CHAR;
367 
368     if (parentPath.find(ROOT_MEDIA_DIR) != 0) {
369         MEDIA_ERR_LOG("invaid path %{private}s, not managed by scanner", path_.c_str());
370         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_DATA},
371             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
372         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
373         return E_DATA;
374     }
375 
376     parentPath.erase(0, len);
377     if (!parentPath.empty()) {
378         data_->SetRelativePath(parentPath);
379         parentPath = string("/") + parentPath.substr(0, parentPath.length() - 1);
380         data_->SetAlbumName(ScannerUtils::GetFileNameFromUri(parentPath));
381     }
382 
383     if (parentId == UNKNOWN_ID) {
384         parentId = mediaScannerDb_->GetIdFromPath(parent);
385         if (parentId == UNKNOWN_ID) {
386             if (parent == ROOT_MEDIA_DIR) {
387                 parentId = 0;
388             } else {
389                 MEDIA_ERR_LOG("failed to get parent id");
390                 VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_DATA},
391                     {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
392                 PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
393                 return E_DATA;
394             }
395         }
396     }
397     data_->SetParentId(parentId);
398 
399     return E_OK;
400 }
401 
ParseLivePhoto(const std::string & path,const std::unique_ptr<Metadata> & data)402 void ParseLivePhoto(const std::string& path, const std::unique_ptr<Metadata>& data)
403 {
404     if (!MediaFileUtils::IsMovingPhotoMimeType(data->GetFileMimeType())) {
405         return;
406     }
407     if (!MovingPhotoFileUtils::IsLivePhoto(path)) {
408         return;
409     }
410 
411     string extraDataDir = MovingPhotoFileUtils::GetMovingPhotoExtraDataDir(path);
412     if (extraDataDir.empty()) {
413         MEDIA_ERR_LOG("failed to get local extra data dir");
414         return;
415     }
416     if (!MediaFileUtils::IsFileExists(extraDataDir) && !MediaFileUtils::CreateDirectory(extraDataDir)) {
417         MEDIA_ERR_LOG("Failed to create file, path:%{private}s", extraDataDir.c_str());
418         return;
419     }
420     if (MovingPhotoFileUtils::ConvertToMovingPhoto(path, path,
421             MovingPhotoFileUtils::GetMovingPhotoVideoPath(path),
422             MovingPhotoFileUtils::GetMovingPhotoExtraDataPath(path)) == E_OK) {
423         size_t imageSize;
424         if (MediaFileUtils::GetFileSize(path, imageSize)) {
425             data->SetFileSize(static_cast<int64_t>(imageSize));
426         }
427         data->SetPhotoSubType(static_cast<int32_t>(PhotoSubType::MOVING_PHOTO));
428     }
429 }
430 
GetFileInfo(const string & path,int64_t & size,int64_t & dateModified)431 static void GetFileInfo(const string &path, int64_t &size, int64_t &dateModified)
432 {
433     struct stat statInfo {};
434     if (stat(path.c_str(), &statInfo) != E_OK) {
435         MEDIA_ERR_LOG("stat error, path: %{public}s, errno: %{public}d", path.c_str(), errno);
436         return;
437     }
438     size = statInfo.st_size;
439     dateModified = MediaFileUtils::Timespec2Millisecond(statInfo.st_mtim);
440 }
441 
GetMovingPhotoFileInfo(const string & imagePath,int64_t & movingPhotoSize,int64_t & movingPhotoDateModified)442 static void GetMovingPhotoFileInfo(const string &imagePath, int64_t &movingPhotoSize,
443     int64_t &movingPhotoDateModified)
444 {
445     int64_t imageSize = 0;
446     int64_t videoSize = 0;
447     size_t extraDataSize = 0;
448     int64_t imageDateModified = 0;
449     int64_t videoDateModified = 0;
450     string videoPath = MovingPhotoFileUtils::GetMovingPhotoVideoPath(imagePath);
451     string extraDataPath = MovingPhotoFileUtils::GetMovingPhotoExtraDataPath(imagePath);
452     GetFileInfo(imagePath, imageSize, imageDateModified);
453     GetFileInfo(videoPath, videoSize, videoDateModified);
454     (void)MediaFileUtils::GetFileSize(extraDataPath, extraDataSize);
455     movingPhotoSize = imageSize + videoSize + static_cast<int64_t>(extraDataSize);
456     movingPhotoDateModified = imageDateModified >= videoDateModified ? imageDateModified : videoDateModified;
457 }
458 
IsFileNotChanged(const unique_ptr<Metadata> & data,const struct stat & statInfo)459 static bool IsFileNotChanged(const unique_ptr<Metadata> &data, const struct stat &statInfo)
460 {
461     int64_t previousDateModified = data->GetFileDateModified();
462     int64_t previousSize = data->GetFileSize();
463     int64_t currentDateModified = MediaFileUtils::Timespec2Millisecond(statInfo.st_mtim);
464     int64_t currentSize = statInfo.st_size;
465     if (data->GetPhotoSubType() == static_cast<int32_t>(PhotoSubType::MOVING_PHOTO) ||
466         data->GetMovingPhotoEffectMode() == static_cast<int32_t>(MovingPhotoEffectMode::IMAGE_ONLY)) {
467         GetMovingPhotoFileInfo(data->GetFilePath(), currentSize, currentDateModified);
468     }
469     return previousDateModified == currentDateModified && previousSize == currentSize;
470 }
471 
BuildData(const struct stat & statInfo)472 int32_t MediaScannerObj::BuildData(const struct stat &statInfo)
473 {
474     data_ = make_unique<Metadata>();
475     if (data_ == nullptr) {
476         MEDIA_ERR_LOG("failed to make unique ptr for metadata");
477         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_DATA},
478             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
479         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
480         return E_DATA;
481     }
482 
483     if (S_ISDIR(statInfo.st_mode)) {
484         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_INVALID_ARGUMENTS},
485             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
486         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
487         return E_INVALID_ARGUMENTS;
488     }
489 
490     int32_t err = mediaScannerDb_->GetFileBasicInfo(path_, data_, api_, fileId_);
491     if (err != E_OK) {
492         MEDIA_ERR_LOG("failed to get file basic info");
493         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
494             {KEY_OPT_TYPE, OptType::SCAN}};
495         PostEventUtils::GetInstance().PostErrorProcess(ErrType::DB_OPT_ERR, map);
496         return err;
497     }
498     if (data_->GetFileDateModified() == 0) {
499         data_->SetForAdd(true);
500     }
501 
502     // file path
503     data_->SetFilePath(path_);
504     // may need isPending here
505     if (IsFileNotChanged(data_, statInfo) && !isForceScan_) {
506         scannedIds_.insert(make_pair(data_->GetTableName(), data_->GetFileId()));
507         if (path_.find(ROOT_MEDIA_DIR + PHOTO_BUCKET) != string::npos) {
508             MEDIA_WARN_LOG("no need to scan, date_modified:%{public}ld, size:%{public}ld, pending:%{public}ld",
509                 static_cast<long>(data_->GetFileDateModified()), static_cast<long>(data_->GetFileSize()),
510                 static_cast<long>(data_->GetTimePending()));
511         }
512         return E_SCANNED;
513     }
514 
515     if (data_->GetFileId() == FILE_ID_DEFAULT) {
516         data_->SetFileName(ScannerUtils::GetFileNameFromUri(path_));
517     }
518     data_->SetFileTitle(ScannerUtils::GetFileTitle(data_->GetFileName()));
519 
520     // statinfo
521     data_->SetFileSize(statInfo.st_size);
522     // Temp file will not be notified. Do not set fileDateModified to keep GetForAdd true
523     if (!data_->GetIsTemp()) {
524         data_->SetFileDateModified(static_cast<int64_t>(MediaFileUtils::Timespec2Millisecond(statInfo.st_mtim)));
525     }
526 
527     // extension and type
528     string extension = ScannerUtils::GetFileExtension(path_);
529     string mimeType = MimeTypeUtils::GetMimeTypeFromExtension(extension);
530     data_->SetFileExtension(extension);
531     data_->SetFileMimeType(mimeType);
532     data_->SetFileMediaType(MimeTypeUtils::GetMediaTypeFromMimeType(mimeType));
533     ParseLivePhoto(path_, data_);
534     return E_OK;
535 }
536 
GetFileMetadata()537 int32_t MediaScannerObj::GetFileMetadata()
538 {
539     if (path_.empty()) {
540         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_INVALID_ARGUMENTS},
541             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
542         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
543         return E_INVALID_ARGUMENTS;
544     }
545     struct stat statInfo = { 0 };
546     if (stat(path_.c_str(), &statInfo) != 0) {
547         MEDIA_ERR_LOG("stat syscall err %{public}d", errno);
548         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_SYSCALL},
549             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
550         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
551         return E_SYSCALL;
552     }
553     if (statInfo.st_size == 0) {
554         MEDIA_WARN_LOG("file size is 0, path: %{private}s", path_.c_str());
555     }
556 
557     int errCode = BuildData(statInfo);
558     if (errCode != E_OK) {
559         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, errCode},
560             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
561         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
562         return errCode;
563     }
564 
565     return E_OK;
566 }
567 
ScanFileInternal()568 int32_t MediaScannerObj::ScanFileInternal()
569 {
570     if (ScannerUtils::IsFileHidden(path_)) {
571         MEDIA_ERR_LOG("the file is hidden");
572         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_FILE_HIDDEN},
573             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
574         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
575         return E_FILE_HIDDEN;
576     }
577 
578     int32_t err = GetFileMetadata();
579     if (err != E_OK) {
580         if (err != E_SCANNED) {
581             MEDIA_ERR_LOG("failed to get file metadata");
582             VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
583                 {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
584             PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
585         }
586         return err;
587     }
588     string parent = ScannerUtils::GetParentPath(path_);
589     err = GetParentDirInfo(parent, UNKNOWN_ID);
590     if (err != E_OK) {
591         MEDIA_ERR_LOG("failed to get dir info");
592         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
593             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
594         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
595         return err;
596     }
597 
598     err = GetMediaInfo();
599     if (err != E_OK) {
600         MEDIA_ERR_LOG("failed to get media info, error: %{public}d", err);
601         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
602             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
603         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
604         // no return here for fs metadata being updated or inserted
605     }
606 
607     err = Commit();
608     if (err != E_OK) {
609         MEDIA_ERR_LOG("failed to commit err %{public}d", err);
610         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
611             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
612         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
613         return err;
614     }
615 
616     return E_OK;
617 }
618 
BuildFileInfo(const string & parent,int32_t parentId)619 int32_t MediaScannerObj::BuildFileInfo(const string &parent, int32_t parentId)
620 {
621     int32_t err = GetParentDirInfo(parent, parentId);
622     if (err != E_OK) {
623         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
624             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
625         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
626         MEDIA_ERR_LOG("failed to get dir info");
627         return err;
628     }
629 
630     err = GetMediaInfo();
631     if (err != E_OK) {
632         MEDIA_ERR_LOG("failed to get media info");
633         // no return here for fs metadata being updated or inserted
634         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
635             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
636         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
637     }
638 
639     err = AddToTransaction();
640     if (err != E_OK) {
641         MEDIA_ERR_LOG("failed to add to transaction err %{public}d", err);
642         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
643             {KEY_OPT_TYPE, OptType::SCAN}};
644         PostEventUtils::GetInstance().PostErrorProcess(ErrType::DB_OPT_ERR, map);
645         return err;
646     }
647 
648     return E_OK;
649 }
650 
ScanFileInTraversal(const string & path,const string & parent,int32_t parentId)651 int32_t MediaScannerObj::ScanFileInTraversal(const string &path, const string &parent, int32_t parentId)
652 {
653     path_ = path;
654     if (ScannerUtils::IsFileHidden(path_)) {
655         MEDIA_ERR_LOG("the file is hidden");
656         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_FILE_HIDDEN},
657             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
658         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
659         return E_FILE_HIDDEN;
660     }
661     int32_t err = GetFileMetadata();
662     if (err != E_OK) {
663         if (err != E_SCANNED) {
664             MEDIA_ERR_LOG("failed to get file metadata");
665             VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
666                 {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
667             PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
668         }
669         return err;
670     }
671 
672     if (data_->GetTimePending() != 0) {
673         MEDIA_INFO_LOG("File %{private}s is pending", path.c_str());
674         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_IS_PENDING},
675             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
676         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
677         return E_IS_PENDING;
678     }
679 
680 #ifdef MEDIALIBRARY_COMPATIBILITY
681     SetPhotoSubType(parent);
682 #endif
683 
684     err = BuildFileInfo(parent, parentId);
685     if (err != E_OK) {
686         MEDIA_ERR_LOG("failed to get other file metadata");
687         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
688             {KEY_OPT_FILE, path_}, {KEY_OPT_TYPE, OptType::SCAN}};
689         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
690         return err;
691     }
692 
693     return E_OK;
694 }
695 
InsertOrUpdateAlbumInfo(const string & albumPath,int32_t parentId,const string & albumName)696 int32_t MediaScannerObj::InsertOrUpdateAlbumInfo(const string &albumPath, int32_t parentId,
697     const string &albumName)
698 {
699     struct stat statInfo;
700     int32_t albumId = UNKNOWN_ID;
701     bool update = false;
702 
703     if (stat(albumPath.c_str(), &statInfo)) {
704         MEDIA_ERR_LOG("stat dir error %{public}d", errno);
705         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, -errno},
706             {KEY_OPT_FILE, albumPath}, {KEY_OPT_TYPE, OptType::SCAN}};
707         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
708         return UNKNOWN_ID;
709     }
710     if (SkipBucket(albumPath)) {
711         return FILE_ID_DEFAULT;
712     }
713     if (albumMap_.find(albumPath) != albumMap_.end()) {
714         Metadata albumInfo = albumMap_.at(albumPath);
715         albumId = albumInfo.GetFileId();
716 
717         if ((albumInfo.GetFileDateModified() / MSEC_TO_SEC) == statInfo.st_mtime) {
718             scannedIds_.insert(make_pair(MEDIALIBRARY_TABLE, albumId));
719             return albumId;
720         } else {
721             update = true;
722         }
723     }
724 
725     Metadata metadata;
726     metadata.SetFilePath(albumPath);
727     metadata.SetFileName(ScannerUtils::GetFileNameFromUri(albumPath));
728     metadata.SetFileTitle(ScannerUtils::GetFileTitle(metadata.GetFileName()));
729     metadata.SetFileMediaType(static_cast<MediaType>(MEDIA_TYPE_ALBUM));
730     metadata.SetFileSize(statInfo.st_size);
731     metadata.SetFileDateModified(MediaFileUtils::Timespec2Millisecond(statInfo.st_mtim));
732 
733     string relativePath = ScannerUtils::GetParentPath(albumPath) + SLASH_CHAR;
734     metadata.SetRelativePath(relativePath.erase(0, ROOT_MEDIA_DIR.length()));
735     metadata.SetParentId(parentId);
736     metadata.SetAlbumName(albumName);
737 
738     if (update) {
739         metadata.SetFileId(albumId);
740         albumId = mediaScannerDb_->UpdateAlbum(metadata);
741     } else {
742         albumId = mediaScannerDb_->InsertAlbum(metadata);
743     }
744     scannedIds_.insert(make_pair(MEDIALIBRARY_TABLE, albumId));
745 
746     return albumId;
747 }
748 
CleanupDirectory()749 int32_t MediaScannerObj::CleanupDirectory()
750 {
751     unordered_set<MediaType> mediaTypeSet;
752     vector<string> scanTables = {
753         MEDIALIBRARY_TABLE, PhotoColumn::PHOTOS_TABLE, AudioColumn::AUDIOS_TABLE
754     };
755 
756     for (auto table : scanTables) {
757         // clean up table
758         vector<string> deleteIdList;
759         unordered_map<int32_t, MediaType> prevIdMap;
760         if (table == PhotoColumn::PHOTOS_TABLE) {
761             prevIdMap = mediaScannerDb_->GetIdsFromFilePath(dir_, PhotoColumn::PHOTOS_TABLE,
762                 ROOT_MEDIA_DIR + PHOTO_BUCKET);
763         } else if (table == AudioColumn::AUDIOS_TABLE) {
764             prevIdMap = mediaScannerDb_->GetIdsFromFilePath(dir_, AudioColumn::AUDIOS_TABLE,
765                 ROOT_MEDIA_DIR + AUDIO_BUCKET);
766         } else {
767             prevIdMap = mediaScannerDb_->GetIdsFromFilePath(dir_, table);
768         }
769 
770         for (auto itr : prevIdMap) {
771             auto it = scannedIds_.find(make_pair(table, itr.first));
772             if (it != scannedIds_.end()) {
773                 scannedIds_.erase(it);
774             } else {
775                 deleteIdList.push_back(to_string(itr.first));
776                 mediaTypeSet.insert(itr.second);
777             }
778         }
779 
780         if (!deleteIdList.empty()) {
781             mediaScannerDb_->DeleteMetadata(deleteIdList, table);
782         }
783     }
784 
785     for (const MediaType &mediaType : mediaTypeSet) {
786         mediaScannerDb_->NotifyDatabaseChange(mediaType);
787     }
788 
789     scannedIds_.clear();
790 
791     return E_OK;
792 }
793 
OpenFailedOperation(const string & path)794 static int32_t OpenFailedOperation(const string &path)
795 {
796     MEDIA_ERR_LOG("Failed to opendir %{private}s, errno %{private}d", path.c_str(), errno);
797     VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, -errno},
798         {KEY_OPT_FILE, path}, {KEY_OPT_TYPE, OptType::SCAN}};
799     PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
800     return ERR_NOT_ACCESSIBLE;
801 }
802 
WalkFileTree(const string & path,int32_t parentId)803 int32_t MediaScannerObj::WalkFileTree(const string &path, int32_t parentId)
804 {
805     int err = E_OK;
806     DIR *dirPath = nullptr;
807     struct dirent *ent = nullptr;
808     size_t len = path.length();
809     struct stat statInfo;
810 
811     if (len >= FILENAME_MAX - 1) {
812         return ERR_INCORRECT_PATH;
813     }
814 
815     auto fName = (char *)calloc(FILENAME_MAX, sizeof(char));
816     if (fName == nullptr) {
817         return ERR_MEM_ALLOC_FAIL;
818     }
819 
820     if (strcpy_s(fName, FILENAME_MAX, path.c_str()) != ERR_SUCCESS) {
821         FREE_MEMORY_AND_SET_NULL(fName);
822         return ERR_MEM_ALLOC_FAIL;
823     }
824     fName[len++] = '/';
825     if ((dirPath = opendir(path.c_str())) == nullptr) {
826         FREE_MEMORY_AND_SET_NULL(fName);
827         return OpenFailedOperation(path);
828     }
829 
830     while ((ent = readdir(dirPath)) != nullptr) {
831         if (*stopFlag_) {
832             err = E_STOP;
833             break;
834         }
835 
836         if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
837             continue;
838         }
839 
840         if (strncpy_s(fName + len, FILENAME_MAX - len, ent->d_name, FILENAME_MAX - len)) {
841             continue;
842         }
843 
844         if (lstat(fName, &statInfo) == -1) {
845             continue;
846         }
847 
848         string currentPath = fName;
849         if (S_ISDIR(statInfo.st_mode)) {
850             if (ScannerUtils::IsDirHidden(currentPath, skipPhoto_)) {
851                 continue;
852             }
853             MEDIA_INFO_LOG("Walk dir %{public}s", currentPath.c_str());
854             int32_t albumId = InsertOrUpdateAlbumInfo(currentPath, parentId, ent->d_name);
855             if (albumId == UNKNOWN_ID) {
856                 err = E_DATA;
857                 // might break in later pr for a rescan
858                 continue;
859             }
860 
861             (void)WalkFileTree(currentPath, albumId);
862         } else {
863             (void)ScanFileInTraversal(currentPath, path, parentId);
864         }
865     }
866 
867     closedir(dirPath);
868     FREE_MEMORY_AND_SET_NULL(fName);
869 
870     return err;
871 }
872 
ScanDirInternal()873 int32_t MediaScannerObj::ScanDirInternal()
874 {
875     if (ScannerUtils::IsDirHiddenRecursive(dir_, skipPhoto_)) {
876         MEDIA_ERR_LOG("the dir %{private}s is hidden", dir_.c_str());
877         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, E_DIR_HIDDEN},
878             {KEY_OPT_FILE, dir_}, {KEY_OPT_TYPE, OptType::SCAN}};
879         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
880         return E_DIR_HIDDEN;
881     }
882 
883     /*
884      * 1. may query albums in batch for the big data case
885      * 2. postpone this operation might avoid some conflicts
886      */
887     int32_t err = mediaScannerDb_->ReadAlbums(dir_, albumMap_);
888     if (err != E_OK) {
889         MEDIA_ERR_LOG("read albums err %{public}d", err);
890         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
891             {KEY_OPT_TYPE, OptType::SCAN}};
892         PostEventUtils::GetInstance().PostErrorProcess(ErrType::DB_OPT_ERR, map);
893         return err;
894     }
895     /* no further operation when stopped */
896     err = WalkFileTree(dir_, NO_PARENT);
897     if (err != E_OK) {
898         MEDIA_ERR_LOG("walk file tree err %{public}d", err);
899         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
900             {KEY_OPT_FILE, dir_}, {KEY_OPT_TYPE, OptType::SCAN}};
901         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
902         return err;
903     }
904     err = CommitTransaction();
905     if (err != E_OK) {
906         MEDIA_ERR_LOG("commit transaction err %{public}d", err);
907         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
908             {KEY_OPT_TYPE, OptType::SCAN}};
909         PostEventUtils::GetInstance().PostErrorProcess(ErrType::DB_OPT_ERR, map);
910         return err;
911     }
912     err = CleanupDirectory();
913     if (err != E_OK) {
914         MEDIA_ERR_LOG("clean up dir err %{public}d", err);
915         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, err},
916             {KEY_OPT_FILE, dir_}, {KEY_OPT_TYPE, OptType::SCAN}};
917         PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
918         return err;
919     }
920 
921     return E_OK;
922 }
923 
Start()924 int32_t MediaScannerObj::Start()
925 {
926     int32_t ret = ScanError(true);
927     if (ret != E_OK) {
928         MEDIA_ERR_LOG("scann error fail %{public}d", ret);
929         return ret;
930     }
931 
932     /*
933      * primary key wouldn't be duplicate
934      */
935     ret = mediaScannerDb_->RecordError(ROOT_MEDIA_DIR);
936     if (ret != E_OK) {
937         MEDIA_ERR_LOG("record err fail %{public}d", ret);
938         VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, ret},
939             {KEY_OPT_TYPE, OptType::SCAN}};
940         PostEventUtils::GetInstance().PostErrorProcess(ErrType::DB_OPT_ERR, map);
941         return ret;
942     }
943 
944     return E_OK;
945 }
946 
ScanError(bool isBoot)947 int32_t MediaScannerObj::ScanError(bool isBoot)
948 {
949     auto errSet = mediaScannerDb_->ReadError();
950     for (auto &err : errSet) {
951         string realPath;
952         if (!PathToRealPath(err, realPath)) {
953             MEDIA_ERR_LOG("failed to get real path %{private}s, errno %{public}d", err.c_str(), errno);
954             VariantMap map = {{KEY_ERR_FILE, __FILE__}, {KEY_ERR_LINE, __LINE__}, {KEY_ERR_CODE, -errno},
955                 {KEY_OPT_FILE, err}, {KEY_OPT_TYPE, OptType::SCAN}};
956             PostEventUtils::GetInstance().PostErrorProcess(ErrType::FILE_OPT_ERR, map);
957             (void)mediaScannerDb_->DeleteError(err);
958             continue;
959         }
960 
961         callback_ = make_shared<ScanErrCallback>(err);
962 
963         /*
964          * Scan full path only when boot; all other errors are processed in
965          * broadcast receving context.
966          */
967         if (err == ROOT_MEDIA_DIR) {
968             if (isBoot) {
969                 dir_ = move(realPath);
970                 (void)ScanDir();
971                 break;
972             } else {
973                 continue;
974             }
975         }
976 
977         /* assume err paths are correct */
978         if (ScannerUtils::IsDirectory(realPath)) {
979             dir_ = move(realPath);
980             (void)ScanDir();
981         } else if (ScannerUtils::IsRegularFile(realPath)) {
982             path_ = move(realPath);
983             (void)ScanFile();
984         }
985     }
986 
987     return E_OK;
988 }
989 
SetError()990 int32_t MediaScannerObj::SetError()
991 {
992     int32_t ret = mediaScannerDb_->RecordError(errorPath_);
993     if (ret != E_OK) {
994         MEDIA_ERR_LOG("record err fail %{public}d", ret);
995         return ret;
996     }
997 
998     return E_OK;
999 }
1000 } // namespace Media
1001 } // namespace OHOS
1002