1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "HardwareBitmapUploader.h"
18
19 #include <EGL/egl.h>
20 #include <EGL/eglext.h>
21 #include <GLES2/gl2.h>
22 #include <GLES2/gl2ext.h>
23 #include <GLES3/gl3.h>
24 #include <GrDirectContext.h>
25 #include <SkBitmap.h>
26 #include <SkCanvas.h>
27 #include <SkImage.h>
28 #include <SkImageInfo.h>
29 #include <SkRefCnt.h>
30 #include <gui/TraceUtils.h>
31 #include <utils/GLUtils.h>
32 #include <utils/NdkUtils.h>
33 #include <utils/Trace.h>
34
35 #include <thread>
36
37 #include "hwui/Bitmap.h"
38 #include "renderthread/EglManager.h"
39 #include "renderthread/VulkanManager.h"
40 #include "thread/ThreadBase.h"
41 #include "utils/TimeUtils.h"
42
43 namespace android::uirenderer {
44
45 static constexpr auto kThreadTimeout = 60000_ms;
46
47 class AHBUploader;
48 // This helper uploader classes allows us to upload using either EGL or Vulkan using the same
49 // interface.
50 static sp<AHBUploader> sUploader = nullptr;
51
52 struct FormatInfo {
53 AHardwareBuffer_Format bufferFormat;
54 GLint format, type;
55 VkFormat vkFormat;
56 bool isSupported = false;
57 bool valid = true;
58 };
59
60 class AHBUploader : public RefBase {
61 public:
~AHBUploader()62 virtual ~AHBUploader() {}
63
destroy()64 void destroy() {
65 std::lock_guard _lock{mLock};
66 LOG_ALWAYS_FATAL_IF(mPendingUploads, "terminate called while uploads in progress");
67 if (mUploadThread) {
68 mUploadThread->requestExit();
69 mUploadThread->join();
70 mUploadThread = nullptr;
71 }
72 onDestroy();
73 }
74
uploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)75 bool uploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
76 AHardwareBuffer* ahb) {
77 ATRACE_CALL();
78 beginUpload();
79 bool result = onUploadHardwareBitmap(bitmap, format, ahb);
80 endUpload();
81 return result;
82 }
83
postIdleTimeoutCheck()84 void postIdleTimeoutCheck() {
85 mUploadThread->queue().postDelayed(kThreadTimeout, [this]() { this->idleTimeoutCheck(); });
86 }
87
88 protected:
89 std::mutex mLock;
90 sp<ThreadBase> mUploadThread = nullptr;
91
92 private:
93 virtual void onIdle() = 0;
94 virtual void onDestroy() = 0;
95
96 virtual bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
97 AHardwareBuffer* ahb) = 0;
98 virtual void onBeginUpload() = 0;
99
shouldTimeOutLocked()100 bool shouldTimeOutLocked() {
101 nsecs_t durationSince = systemTime() - mLastUpload;
102 return durationSince > kThreadTimeout;
103 }
104
idleTimeoutCheck()105 void idleTimeoutCheck() {
106 std::lock_guard _lock{mLock};
107 if (mPendingUploads == 0 && shouldTimeOutLocked()) {
108 onIdle();
109 } else {
110 this->postIdleTimeoutCheck();
111 }
112 }
113
beginUpload()114 void beginUpload() {
115 std::lock_guard _lock{mLock};
116 mPendingUploads++;
117
118 if (!mUploadThread) {
119 mUploadThread = new ThreadBase{};
120 }
121 if (!mUploadThread->isRunning()) {
122 mUploadThread->start("GrallocUploadThread");
123 }
124
125 onBeginUpload();
126 }
127
endUpload()128 void endUpload() {
129 std::lock_guard _lock{mLock};
130 mPendingUploads--;
131 mLastUpload = systemTime();
132 }
133
134 int mPendingUploads = 0;
135 nsecs_t mLastUpload = 0;
136 };
137
138 #define FENCE_TIMEOUT 2000000000
139
140 class EGLUploader : public AHBUploader {
141 private:
onDestroy()142 void onDestroy() override {
143 mEglManager.destroy();
144 }
onIdle()145 void onIdle() override {
146 mEglManager.destroy();
147 }
148
onBeginUpload()149 void onBeginUpload() override {
150 if (!mEglManager.hasEglContext()) {
151 mUploadThread->queue().runSync([this]() {
152 this->mEglManager.initialize();
153 glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
154 });
155
156 this->postIdleTimeoutCheck();
157 }
158 }
159
160
getUploadEglDisplay()161 EGLDisplay getUploadEglDisplay() {
162 std::lock_guard _lock{mLock};
163 LOG_ALWAYS_FATAL_IF(!mEglManager.hasEglContext(), "Forgot to begin an upload?");
164 return mEglManager.eglDisplay();
165 }
166
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)167 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
168 AHardwareBuffer* ahb) override {
169 ATRACE_CALL();
170
171 EGLDisplay display = getUploadEglDisplay();
172
173 LOG_ALWAYS_FATAL_IF(display == EGL_NO_DISPLAY, "Failed to get EGL_DEFAULT_DISPLAY! err=%s",
174 uirenderer::renderthread::EglManager::eglErrorString());
175 // We use an EGLImage to access the content of the buffer
176 // The EGL image is later bound to a 2D texture
177 const EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(ahb);
178 AutoEglImage autoImage(display, clientBuffer);
179 if (autoImage.image == EGL_NO_IMAGE_KHR) {
180 ALOGW("Could not create EGL image, err =%s",
181 uirenderer::renderthread::EglManager::eglErrorString());
182 return false;
183 }
184
185 {
186 ATRACE_FORMAT("CPU -> gralloc transfer (%dx%d)", bitmap.width(), bitmap.height());
187 EGLSyncKHR fence = mUploadThread->queue().runSync([&]() -> EGLSyncKHR {
188 AutoSkiaGlTexture glTexture;
189 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, autoImage.image);
190 if (GLUtils::dumpGLErrors()) {
191 return EGL_NO_SYNC_KHR;
192 }
193
194 // glTexSubImage2D is synchronous in sense that it memcpy() from pointer that we
195 // provide.
196 // But asynchronous in sense that driver may upload texture onto hardware buffer
197 // when we first use it in drawing
198 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, bitmap.width(), bitmap.height(),
199 format.format, format.type, bitmap.getPixels());
200 if (GLUtils::dumpGLErrors()) {
201 return EGL_NO_SYNC_KHR;
202 }
203
204 EGLSyncKHR uploadFence =
205 eglCreateSyncKHR(eglGetCurrentDisplay(), EGL_SYNC_FENCE_KHR, NULL);
206 if (uploadFence == EGL_NO_SYNC_KHR) {
207 ALOGW("Could not create sync fence %#x", eglGetError());
208 };
209 glFlush();
210 GLUtils::dumpGLErrors();
211 return uploadFence;
212 });
213
214 if (fence == EGL_NO_SYNC_KHR) {
215 return false;
216 }
217 EGLint waitStatus = eglClientWaitSyncKHR(display, fence, 0, FENCE_TIMEOUT);
218 ALOGE_IF(waitStatus != EGL_CONDITION_SATISFIED_KHR,
219 "Failed to wait for the fence %#x", eglGetError());
220
221 eglDestroySyncKHR(display, fence);
222 }
223 return true;
224 }
225
226 renderthread::EglManager mEglManager;
227 };
228
229 class VkUploader : public AHBUploader {
230 private:
onDestroy()231 void onDestroy() override {
232 std::lock_guard _lock{mVkLock};
233 mGrContext.reset();
234 mVulkanManagerStrong.clear();
235 }
onIdle()236 void onIdle() override {
237 onDestroy();
238 }
239
onBeginUpload()240 void onBeginUpload() override {}
241
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)242 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
243 AHardwareBuffer* ahb) override {
244 bool uploadSucceeded = false;
245 mUploadThread->queue().runSync([this, &uploadSucceeded, bitmap, ahb]() {
246 ATRACE_CALL();
247 std::lock_guard _lock{mVkLock};
248
249 renderthread::VulkanManager* vkManager = getVulkanManager();
250 if (!vkManager->hasVkContext()) {
251 LOG_ALWAYS_FATAL_IF(mGrContext,
252 "GrContext exists with no VulkanManager for vulkan uploads");
253 vkManager->initialize();
254 }
255
256 if (!mGrContext) {
257 GrContextOptions options;
258 mGrContext = vkManager->createContext(options,
259 renderthread::VulkanManager::ContextType::kUploadThread);
260 LOG_ALWAYS_FATAL_IF(!mGrContext, "failed to create GrContext for vulkan uploads");
261 this->postIdleTimeoutCheck();
262 }
263
264 sk_sp<SkImage> image =
265 SkImage::MakeFromAHardwareBufferWithData(mGrContext.get(), bitmap.pixmap(), ahb);
266 mGrContext->submit(true);
267
268 uploadSucceeded = (image.get() != nullptr);
269 });
270 return uploadSucceeded;
271 }
272
273 /* must be called on the upload thread after the vkLock has been acquired */
getVulkanManager()274 renderthread::VulkanManager* getVulkanManager() {
275 if (!mVulkanManagerStrong) {
276 mVulkanManagerStrong = mVulkanManagerWeak.promote();
277
278 // create a new manager if we couldn't promote the weak ref
279 if (!mVulkanManagerStrong) {
280 mVulkanManagerStrong = renderthread::VulkanManager::getInstance();
281 mGrContext.reset();
282 mVulkanManagerWeak = mVulkanManagerStrong;
283 }
284 }
285
286 return mVulkanManagerStrong.get();
287 }
288
289 sk_sp<GrDirectContext> mGrContext;
290 sp<renderthread::VulkanManager> mVulkanManagerStrong;
291 wp<renderthread::VulkanManager> mVulkanManagerWeak;
292 std::mutex mVkLock;
293 };
294
checkSupport(AHardwareBuffer_Format format)295 static bool checkSupport(AHardwareBuffer_Format format) {
296 AHardwareBuffer_Desc desc = {
297 .width = 1,
298 .height = 1,
299 .layers = 1,
300 .format = format,
301 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
302 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
303 };
304 UniqueAHardwareBuffer buffer = allocateAHardwareBuffer(desc);
305 return buffer != nullptr;
306 }
307
hasFP16Support()308 bool HardwareBitmapUploader::hasFP16Support() {
309 static bool hasFP16Support = checkSupport(AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT);
310 return hasFP16Support;
311 }
312
has1010102Support()313 bool HardwareBitmapUploader::has1010102Support() {
314 static bool has101012Support = checkSupport(AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM);
315 return has101012Support;
316 }
317
hasAlpha8Support()318 bool HardwareBitmapUploader::hasAlpha8Support() {
319 static bool hasAlpha8Support = checkSupport(AHARDWAREBUFFER_FORMAT_R8_UNORM);
320 return hasAlpha8Support;
321 }
322
determineFormat(const SkBitmap & skBitmap,bool usingGL)323 static FormatInfo determineFormat(const SkBitmap& skBitmap, bool usingGL) {
324 FormatInfo formatInfo;
325 switch (skBitmap.info().colorType()) {
326 case kRGBA_8888_SkColorType:
327 formatInfo.isSupported = true;
328 [[fallthrough]];
329 // ARGB_4444 is upconverted to RGBA_8888
330 case kARGB_4444_SkColorType:
331 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
332 formatInfo.format = GL_RGBA;
333 formatInfo.type = GL_UNSIGNED_BYTE;
334 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
335 break;
336 case kRGBA_F16_SkColorType:
337 formatInfo.isSupported = HardwareBitmapUploader::hasFP16Support();
338 if (formatInfo.isSupported) {
339 formatInfo.type = GL_HALF_FLOAT;
340 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT;
341 formatInfo.vkFormat = VK_FORMAT_R16G16B16A16_SFLOAT;
342 } else {
343 formatInfo.type = GL_UNSIGNED_BYTE;
344 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
345 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
346 }
347 formatInfo.format = GL_RGBA;
348 break;
349 case kRGB_565_SkColorType:
350 formatInfo.isSupported = true;
351 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM;
352 formatInfo.format = GL_RGB;
353 formatInfo.type = GL_UNSIGNED_SHORT_5_6_5;
354 formatInfo.vkFormat = VK_FORMAT_R5G6B5_UNORM_PACK16;
355 break;
356 case kGray_8_SkColorType:
357 formatInfo.isSupported = usingGL;
358 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
359 formatInfo.format = GL_LUMINANCE;
360 formatInfo.type = GL_UNSIGNED_BYTE;
361 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
362 break;
363 case kRGBA_1010102_SkColorType:
364 formatInfo.isSupported = HardwareBitmapUploader::has1010102Support();
365 if (formatInfo.isSupported) {
366 formatInfo.type = GL_UNSIGNED_INT_2_10_10_10_REV;
367 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM;
368 formatInfo.vkFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
369 } else {
370 formatInfo.type = GL_UNSIGNED_BYTE;
371 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
372 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
373 }
374 formatInfo.format = GL_RGBA;
375 break;
376 case kAlpha_8_SkColorType:
377 formatInfo.isSupported = HardwareBitmapUploader::hasAlpha8Support();
378 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8_UNORM;
379 formatInfo.format = GL_R8;
380 formatInfo.type = GL_UNSIGNED_BYTE;
381 formatInfo.vkFormat = VK_FORMAT_R8_UNORM;
382 break;
383 default:
384 ALOGW("unable to create hardware bitmap of colortype: %d", skBitmap.info().colorType());
385 formatInfo.valid = false;
386 }
387 return formatInfo;
388 }
389
makeHwCompatible(const FormatInfo & format,const SkBitmap & source)390 static SkBitmap makeHwCompatible(const FormatInfo& format, const SkBitmap& source) {
391 if (format.isSupported) {
392 return source;
393 } else {
394 SkBitmap bitmap;
395 bitmap.allocPixels(source.info().makeColorType(kN32_SkColorType));
396 bitmap.writePixels(source.pixmap());
397 return bitmap;
398 }
399 }
400
401
createUploader(bool usingGL)402 static void createUploader(bool usingGL) {
403 static std::mutex lock;
404 std::lock_guard _lock{lock};
405 if (!sUploader.get()) {
406 if (usingGL) {
407 sUploader = new EGLUploader();
408 } else {
409 sUploader = new VkUploader();
410 }
411 }
412 }
413
allocateHardwareBitmap(const SkBitmap & sourceBitmap)414 sk_sp<Bitmap> HardwareBitmapUploader::allocateHardwareBitmap(const SkBitmap& sourceBitmap) {
415 ATRACE_CALL();
416
417 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
418 uirenderer::RenderPipelineType::SkiaGL;
419
420 FormatInfo format = determineFormat(sourceBitmap, usingGL);
421 if (!format.valid) {
422 return nullptr;
423 }
424
425 SkBitmap bitmap = makeHwCompatible(format, sourceBitmap);
426 AHardwareBuffer_Desc desc = {
427 .width = static_cast<uint32_t>(bitmap.width()),
428 .height = static_cast<uint32_t>(bitmap.height()),
429 .layers = 1,
430 .format = format.bufferFormat,
431 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
432 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
433 };
434 UniqueAHardwareBuffer ahb = allocateAHardwareBuffer(desc);
435 if (!ahb) {
436 ALOGW("allocateHardwareBitmap() failed in AHardwareBuffer_allocate()");
437 return nullptr;
438 };
439
440 createUploader(usingGL);
441
442 if (!sUploader->uploadHardwareBitmap(bitmap, format, ahb.get())) {
443 return nullptr;
444 }
445 return Bitmap::createFrom(ahb.get(), bitmap.colorType(), bitmap.refColorSpace(),
446 bitmap.alphaType(), Bitmap::computePalette(bitmap));
447 }
448
initialize()449 void HardwareBitmapUploader::initialize() {
450 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
451 uirenderer::RenderPipelineType::SkiaGL;
452 createUploader(usingGL);
453 }
454
terminate()455 void HardwareBitmapUploader::terminate() {
456 if (sUploader) {
457 sUploader->destroy();
458 }
459 }
460
461 } // namespace android::uirenderer
462