1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "HardwareBitmapUploader.h"
18
19 #include <EGL/egl.h>
20 #include <EGL/eglext.h>
21 #include <GLES2/gl2.h>
22 #include <GLES2/gl2ext.h>
23 #include <GLES3/gl3.h>
24 #include <GrDirectContext.h>
25 #include <SkCanvas.h>
26 #include <SkImage.h>
27 #include <gui/TraceUtils.h>
28 #include <utils/GLUtils.h>
29 #include <utils/NdkUtils.h>
30 #include <utils/Trace.h>
31
32 #include <thread>
33
34 #include "hwui/Bitmap.h"
35 #include "renderthread/EglManager.h"
36 #include "renderthread/VulkanManager.h"
37 #include "thread/ThreadBase.h"
38 #include "utils/TimeUtils.h"
39
40 namespace android::uirenderer {
41
42 class AHBUploader;
43 // This helper uploader classes allows us to upload using either EGL or Vulkan using the same
44 // interface.
45 static sp<AHBUploader> sUploader = nullptr;
46
47 struct FormatInfo {
48 AHardwareBuffer_Format bufferFormat;
49 GLint format, type;
50 VkFormat vkFormat;
51 bool isSupported = false;
52 bool valid = true;
53 };
54
55 class AHBUploader : public RefBase {
56 public:
~AHBUploader()57 virtual ~AHBUploader() {}
58
destroy()59 void destroy() {
60 std::lock_guard _lock{mLock};
61 LOG_ALWAYS_FATAL_IF(mPendingUploads, "terminate called while uploads in progress");
62 if (mUploadThread) {
63 mUploadThread->requestExit();
64 mUploadThread->join();
65 mUploadThread = nullptr;
66 }
67 onDestroy();
68 }
69
uploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)70 bool uploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
71 AHardwareBuffer* ahb) {
72 ATRACE_CALL();
73 beginUpload();
74 bool result = onUploadHardwareBitmap(bitmap, format, ahb);
75 endUpload();
76 return result;
77 }
78
postIdleTimeoutCheck()79 void postIdleTimeoutCheck() {
80 mUploadThread->queue().postDelayed(5000_ms, [this](){ this->idleTimeoutCheck(); });
81 }
82
83 protected:
84 std::mutex mLock;
85 sp<ThreadBase> mUploadThread = nullptr;
86
87 private:
88 virtual void onIdle() = 0;
89 virtual void onDestroy() = 0;
90
91 virtual bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
92 AHardwareBuffer* ahb) = 0;
93 virtual void onBeginUpload() = 0;
94
shouldTimeOutLocked()95 bool shouldTimeOutLocked() {
96 nsecs_t durationSince = systemTime() - mLastUpload;
97 return durationSince > 2000_ms;
98 }
99
idleTimeoutCheck()100 void idleTimeoutCheck() {
101 std::lock_guard _lock{mLock};
102 if (mPendingUploads == 0 && shouldTimeOutLocked()) {
103 onIdle();
104 } else {
105 this->postIdleTimeoutCheck();
106 }
107 }
108
beginUpload()109 void beginUpload() {
110 std::lock_guard _lock{mLock};
111 mPendingUploads++;
112
113 if (!mUploadThread) {
114 mUploadThread = new ThreadBase{};
115 }
116 if (!mUploadThread->isRunning()) {
117 mUploadThread->start("GrallocUploadThread");
118 }
119
120 onBeginUpload();
121 }
122
endUpload()123 void endUpload() {
124 std::lock_guard _lock{mLock};
125 mPendingUploads--;
126 mLastUpload = systemTime();
127 }
128
129 int mPendingUploads = 0;
130 nsecs_t mLastUpload = 0;
131 };
132
133 #define FENCE_TIMEOUT 2000000000
134
135 class EGLUploader : public AHBUploader {
136 private:
onDestroy()137 void onDestroy() override {
138 mEglManager.destroy();
139 }
onIdle()140 void onIdle() override {
141 mEglManager.destroy();
142 }
143
onBeginUpload()144 void onBeginUpload() override {
145 if (!mEglManager.hasEglContext()) {
146 mUploadThread->queue().runSync([this]() {
147 this->mEglManager.initialize();
148 glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
149 });
150
151 this->postIdleTimeoutCheck();
152 }
153 }
154
155
getUploadEglDisplay()156 EGLDisplay getUploadEglDisplay() {
157 std::lock_guard _lock{mLock};
158 LOG_ALWAYS_FATAL_IF(!mEglManager.hasEglContext(), "Forgot to begin an upload?");
159 return mEglManager.eglDisplay();
160 }
161
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)162 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
163 AHardwareBuffer* ahb) override {
164 ATRACE_CALL();
165
166 EGLDisplay display = getUploadEglDisplay();
167
168 LOG_ALWAYS_FATAL_IF(display == EGL_NO_DISPLAY, "Failed to get EGL_DEFAULT_DISPLAY! err=%s",
169 uirenderer::renderthread::EglManager::eglErrorString());
170 // We use an EGLImage to access the content of the buffer
171 // The EGL image is later bound to a 2D texture
172 const EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(ahb);
173 AutoEglImage autoImage(display, clientBuffer);
174 if (autoImage.image == EGL_NO_IMAGE_KHR) {
175 ALOGW("Could not create EGL image, err =%s",
176 uirenderer::renderthread::EglManager::eglErrorString());
177 return false;
178 }
179
180 {
181 ATRACE_FORMAT("CPU -> gralloc transfer (%dx%d)", bitmap.width(), bitmap.height());
182 EGLSyncKHR fence = mUploadThread->queue().runSync([&]() -> EGLSyncKHR {
183 AutoSkiaGlTexture glTexture;
184 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, autoImage.image);
185 if (GLUtils::dumpGLErrors()) {
186 return EGL_NO_SYNC_KHR;
187 }
188
189 // glTexSubImage2D is synchronous in sense that it memcpy() from pointer that we
190 // provide.
191 // But asynchronous in sense that driver may upload texture onto hardware buffer
192 // when we first use it in drawing
193 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, bitmap.width(), bitmap.height(),
194 format.format, format.type, bitmap.getPixels());
195 if (GLUtils::dumpGLErrors()) {
196 return EGL_NO_SYNC_KHR;
197 }
198
199 EGLSyncKHR uploadFence =
200 eglCreateSyncKHR(eglGetCurrentDisplay(), EGL_SYNC_FENCE_KHR, NULL);
201 if (uploadFence == EGL_NO_SYNC_KHR) {
202 ALOGW("Could not create sync fence %#x", eglGetError());
203 };
204 glFlush();
205 GLUtils::dumpGLErrors();
206 return uploadFence;
207 });
208
209 if (fence == EGL_NO_SYNC_KHR) {
210 return false;
211 }
212 EGLint waitStatus = eglClientWaitSyncKHR(display, fence, 0, FENCE_TIMEOUT);
213 ALOGE_IF(waitStatus != EGL_CONDITION_SATISFIED_KHR,
214 "Failed to wait for the fence %#x", eglGetError());
215
216 eglDestroySyncKHR(display, fence);
217 }
218 return true;
219 }
220
221 renderthread::EglManager mEglManager;
222 };
223
224 class VkUploader : public AHBUploader {
225 private:
onDestroy()226 void onDestroy() override {
227 std::lock_guard _lock{mVkLock};
228 mGrContext.reset();
229 mVulkanManagerStrong.clear();
230 }
onIdle()231 void onIdle() override {
232 onDestroy();
233 }
234
onBeginUpload()235 void onBeginUpload() override {}
236
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)237 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
238 AHardwareBuffer* ahb) override {
239 bool uploadSucceeded = false;
240 mUploadThread->queue().runSync([this, &uploadSucceeded, bitmap, ahb]() {
241 ATRACE_CALL();
242 std::lock_guard _lock{mVkLock};
243
244 renderthread::VulkanManager* vkManager = getVulkanManager();
245 if (!vkManager->hasVkContext()) {
246 LOG_ALWAYS_FATAL_IF(mGrContext,
247 "GrContext exists with no VulkanManager for vulkan uploads");
248 vkManager->initialize();
249 }
250
251 if (!mGrContext) {
252 GrContextOptions options;
253 mGrContext = vkManager->createContext(options,
254 renderthread::VulkanManager::ContextType::kUploadThread);
255 LOG_ALWAYS_FATAL_IF(!mGrContext, "failed to create GrContext for vulkan uploads");
256 this->postIdleTimeoutCheck();
257 }
258
259 sk_sp<SkImage> image =
260 SkImage::MakeFromAHardwareBufferWithData(mGrContext.get(), bitmap.pixmap(), ahb);
261 mGrContext->submit(true);
262
263 uploadSucceeded = (image.get() != nullptr);
264 });
265 return uploadSucceeded;
266 }
267
268 /* must be called on the upload thread after the vkLock has been acquired */
getVulkanManager()269 renderthread::VulkanManager* getVulkanManager() {
270 if (!mVulkanManagerStrong) {
271 mVulkanManagerStrong = mVulkanManagerWeak.promote();
272
273 // create a new manager if we couldn't promote the weak ref
274 if (!mVulkanManagerStrong) {
275 mVulkanManagerStrong = renderthread::VulkanManager::getInstance();
276 mGrContext.reset();
277 mVulkanManagerWeak = mVulkanManagerStrong;
278 }
279 }
280
281 return mVulkanManagerStrong.get();
282 }
283
284 sk_sp<GrDirectContext> mGrContext;
285 sp<renderthread::VulkanManager> mVulkanManagerStrong;
286 wp<renderthread::VulkanManager> mVulkanManagerWeak;
287 std::mutex mVkLock;
288 };
289
hasFP16Support()290 bool HardwareBitmapUploader::hasFP16Support() {
291 static std::once_flag sOnce;
292 static bool hasFP16Support = false;
293
294 // Gralloc shouldn't let us create a USAGE_HW_TEXTURE if GLES is unable to consume it, so
295 // we don't need to double-check the GLES version/extension.
296 std::call_once(sOnce, []() {
297 AHardwareBuffer_Desc desc = {
298 .width = 1,
299 .height = 1,
300 .layers = 1,
301 .format = AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT,
302 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER |
303 AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
304 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
305 };
306 UniqueAHardwareBuffer buffer = allocateAHardwareBuffer(desc);
307 hasFP16Support = buffer != nullptr;
308 });
309
310 return hasFP16Support;
311 }
312
determineFormat(const SkBitmap & skBitmap,bool usingGL)313 static FormatInfo determineFormat(const SkBitmap& skBitmap, bool usingGL) {
314 FormatInfo formatInfo;
315 switch (skBitmap.info().colorType()) {
316 case kRGBA_8888_SkColorType:
317 formatInfo.isSupported = true;
318 [[fallthrough]];
319 // ARGB_4444 is upconverted to RGBA_8888
320 case kARGB_4444_SkColorType:
321 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
322 formatInfo.format = GL_RGBA;
323 formatInfo.type = GL_UNSIGNED_BYTE;
324 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
325 break;
326 case kRGBA_F16_SkColorType:
327 formatInfo.isSupported = HardwareBitmapUploader::hasFP16Support();
328 if (formatInfo.isSupported) {
329 formatInfo.type = GL_HALF_FLOAT;
330 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT;
331 formatInfo.vkFormat = VK_FORMAT_R16G16B16A16_SFLOAT;
332 } else {
333 formatInfo.type = GL_UNSIGNED_BYTE;
334 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
335 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
336 }
337 formatInfo.format = GL_RGBA;
338 break;
339 case kRGB_565_SkColorType:
340 formatInfo.isSupported = true;
341 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM;
342 formatInfo.format = GL_RGB;
343 formatInfo.type = GL_UNSIGNED_SHORT_5_6_5;
344 formatInfo.vkFormat = VK_FORMAT_R5G6B5_UNORM_PACK16;
345 break;
346 case kGray_8_SkColorType:
347 formatInfo.isSupported = usingGL;
348 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
349 formatInfo.format = GL_LUMINANCE;
350 formatInfo.type = GL_UNSIGNED_BYTE;
351 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
352 break;
353 default:
354 ALOGW("unable to create hardware bitmap of colortype: %d", skBitmap.info().colorType());
355 formatInfo.valid = false;
356 }
357 return formatInfo;
358 }
359
makeHwCompatible(const FormatInfo & format,const SkBitmap & source)360 static SkBitmap makeHwCompatible(const FormatInfo& format, const SkBitmap& source) {
361 if (format.isSupported) {
362 return source;
363 } else {
364 SkBitmap bitmap;
365 bitmap.allocPixels(source.info().makeColorType(kN32_SkColorType));
366 bitmap.writePixels(source.pixmap());
367 return bitmap;
368 }
369 }
370
371
createUploader(bool usingGL)372 static void createUploader(bool usingGL) {
373 static std::mutex lock;
374 std::lock_guard _lock{lock};
375 if (!sUploader.get()) {
376 if (usingGL) {
377 sUploader = new EGLUploader();
378 } else {
379 sUploader = new VkUploader();
380 }
381 }
382 }
383
allocateHardwareBitmap(const SkBitmap & sourceBitmap)384 sk_sp<Bitmap> HardwareBitmapUploader::allocateHardwareBitmap(const SkBitmap& sourceBitmap) {
385 ATRACE_CALL();
386
387 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
388 uirenderer::RenderPipelineType::SkiaGL;
389
390 FormatInfo format = determineFormat(sourceBitmap, usingGL);
391 if (!format.valid) {
392 return nullptr;
393 }
394
395 SkBitmap bitmap = makeHwCompatible(format, sourceBitmap);
396 AHardwareBuffer_Desc desc = {
397 .width = static_cast<uint32_t>(bitmap.width()),
398 .height = static_cast<uint32_t>(bitmap.height()),
399 .layers = 1,
400 .format = format.bufferFormat,
401 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
402 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
403 };
404 UniqueAHardwareBuffer ahb = allocateAHardwareBuffer(desc);
405 if (!ahb) {
406 ALOGW("allocateHardwareBitmap() failed in AHardwareBuffer_allocate()");
407 return nullptr;
408 };
409
410 createUploader(usingGL);
411
412 if (!sUploader->uploadHardwareBitmap(bitmap, format, ahb.get())) {
413 return nullptr;
414 }
415 return Bitmap::createFrom(ahb.get(), bitmap.colorType(), bitmap.refColorSpace(),
416 bitmap.alphaType(), Bitmap::computePalette(bitmap));
417 }
418
initialize()419 void HardwareBitmapUploader::initialize() {
420 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
421 uirenderer::RenderPipelineType::SkiaGL;
422 createUploader(usingGL);
423 }
424
terminate()425 void HardwareBitmapUploader::terminate() {
426 if (sUploader) {
427 sUploader->destroy();
428 }
429 }
430
431 } // namespace android::uirenderer
432