diff --git a/app/streaming/video/ffmpeg-renderers/drm.cpp b/app/streaming/video/ffmpeg-renderers/drm.cpp index feac9988..0ba541fc 100644 --- a/app/streaming/video/ffmpeg-renderers/drm.cpp +++ b/app/streaming/video/ffmpeg-renderers/drm.cpp @@ -277,6 +277,15 @@ bool DrmRenderer::prepareDecoderContext(AVCodecContext* context, AVDictionary** return true; } +bool DrmRenderer::prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat) +{ +#ifdef HAVE_EGL + // The surface pool is being reset, so clear the cached EGLImages + m_EglImageFactory.resetCache(); +#endif + return true; +} + void DrmRenderer::prepareToRender() { // Retake DRM master if we dropped it earlier @@ -1485,8 +1494,7 @@ ssize_t DrmRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy, return -1; } - AVDRMFrameDescriptor* drmFrame = (AVDRMFrameDescriptor*)frame->data[0]; - return m_EglImageFactory.exportDRMImages(frame, drmFrame, dpy, images); + return m_EglImageFactory.exportDRMImages(frame, dpy, images); } void DrmRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { diff --git a/app/streaming/video/ffmpeg-renderers/drm.h b/app/streaming/video/ffmpeg-renderers/drm.h index 7b26fe63..b4952a01 100644 --- a/app/streaming/video/ffmpeg-renderers/drm.h +++ b/app/streaming/video/ffmpeg-renderers/drm.h @@ -54,6 +54,7 @@ public: virtual ~DrmRenderer() override; virtual bool initialize(PDECODER_PARAMETERS params) override; virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override; + virtual bool prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat) override; virtual void prepareToRender() override; virtual void renderFrame(AVFrame* frame) override; virtual enum AVPixelFormat getPreferredPixelFormat(int videoFormat) override; diff --git a/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp b/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp index f4221091..fe9c26eb 100644 --- a/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp +++ b/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp @@ -1,5 +1,10 @@ #include "eglimagefactory.h" +#ifdef HAVE_LIBVA +#include +#include +#endif + #include // Don't take a dependency on libdrm just for these constants @@ -21,6 +26,7 @@ EglImageFactory::EglImageFactory(IFFmpegRenderer* renderer) : m_Renderer(renderer), + m_CacheDisabled(false), m_EGLExtDmaBuf(false), m_eglCreateImage(nullptr), m_eglDestroyImage(nullptr), @@ -62,12 +68,76 @@ bool EglImageFactory::initializeEGL(EGLDisplay, return true; } +void EglImageFactory::resetCache() +{ + m_CachedImages.clear(); +} + +ssize_t EglImageFactory::queryImageCache(AVFrame *frame, EGLImage images[EGL_MAX_PLANES]) +{ + if (m_CacheDisabled) { + return -1; + } + else if (!frame->hw_frames_ctx) { + // If we don't have a AVHWFramesContext, we won't have a frame pool + // which means our caching logic of checking AVBuffer pointers won't + // work properly. + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "EGLImage caching disabled due to missing AVHWFramesContext"); + m_CacheDisabled = true; + return -1; + } + else if (!frame->buf[0]) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "EGLImage caching disabled due to missing AVBufferRef"); + m_CacheDisabled = true; + return -1; + } + else if (!frame->buf[0]->buffer) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "EGLImage caching disabled due to missing AVBuffer"); + m_CacheDisabled = true; + return -1; + } + else if (m_CachedImages.size() >= 20) { + // This is a final fail-safe for the case where the buffers aren't + // actually being reused to avoid the cache size growing forever. + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "EGLImage caching disabled due to excessive cached image count"); + m_CacheDisabled = true; + return -1; + } + + auto imgCtx = m_CachedImages.find(frame->buf[0]->buffer); + if (imgCtx == m_CachedImages.end()) { + return -1; + } + + memcpy(images, imgCtx->second.images, sizeof(EGLImage) * imgCtx->second.count); + return imgCtx->second.count; +} + +void EglImageFactory::populateImageCache(AVFrame *frame, EglImageContext &&imgCtx) +{ + AVBuffer* cacheKey = m_CacheDisabled ? nullptr : frame->buf[0]->buffer; + + // Move this entry into the cache + m_CachedImages.emplace(cacheKey, std::move(imgCtx)); +} + #ifdef HAVE_DRM -ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) +ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES); + // Check the cache first + if (ssize_t count = queryImageCache(frame, images); count > 0) { + return count; + } + + AVDRMFrameDescriptor* drmFrame = (AVDRMFrameDescriptor*)frame->data[0]; + // DRM requires composed layers rather than separate layers per plane SDL_assert(drmFrame->nb_layers == 1); @@ -207,12 +277,14 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d attribs[attribIndex++] = EGL_NONE; SDL_assert(attribIndex <= MAX_ATTRIB_COUNT); + EglImageContext imgCtx(dpy, m_eglDestroyImage, m_eglDestroyImageKHR); + // Our EGLImages are non-planar, so we only populate the first entry if (m_eglCreateImage) { - images[0] = m_eglCreateImage(dpy, EGL_NO_CONTEXT, - EGL_LINUX_DMA_BUF_EXT, - nullptr, attribs); - if (!images[0]) { + imgCtx.images[0] = m_eglCreateImage(dpy, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + nullptr, attribs); + if (!imgCtx.images[0]) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "eglCreateImage() Failed: %d", eglGetError()); return -1; @@ -225,16 +297,24 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d intAttribs[i] = (EGLint)attribs[i]; } - images[0] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT, - EGL_LINUX_DMA_BUF_EXT, - nullptr, intAttribs); - if (!images[0]) { + imgCtx.images[0] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + nullptr, intAttribs); + if (!imgCtx.images[0]) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "eglCreateImageKHR() Failed: %d", eglGetError()); return -1; } } + imgCtx.count = 1; + + // Copy the output from the image context before we move it + images[0] = imgCtx.images[0]; + + // Move this image context into the cache + populateImageCache(frame, std::move(imgCtx)); + return 1; } @@ -242,16 +322,45 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d #ifdef HAVE_LIBVA -ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescriptor *vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) +ssize_t EglImageFactory::exportVAImages(AVFrame *frame, uint32_t exportFlags, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { - ssize_t count = 0; - memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES); - SDL_assert(vaFrame->num_layers <= EGL_MAX_PLANES); + auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data; + AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)hwFrameCtx->device_ctx->hwctx; + VASurfaceID surface_id = (VASurfaceID)(uintptr_t)frame->data[3]; - for (size_t i = 0; i < vaFrame->num_layers; ++i) { - const auto &layer = vaFrame->layers[i]; + // Sync the surface before doing anything. We need to do this even if we've got cached EGLImages. + VAStatus st = vaSyncSurface(vaDeviceContext->display, surface_id); + if (st != VA_STATUS_SUCCESS) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "vaSyncSurface() failed: %d", st); + return -1; + } + + // Check the cache first + if (ssize_t count = queryImageCache(frame, images); count > 0) { + return count; + } + + EglImageContext imgCtx(dpy, m_eglDestroyImage, m_eglDestroyImageKHR); + + VADRMPRIMESurfaceDescriptor vaFrame; + st = vaExportSurfaceHandle(vaDeviceContext->display, + surface_id, + VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, + exportFlags, + &vaFrame); + if (st != VA_STATUS_SUCCESS) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "vaExportSurfaceHandle failed: %d", st); + return -1; + } + + SDL_assert(vaFrame.num_layers <= EGL_MAX_PLANES); + + for (size_t i = 0; i < vaFrame.num_layers; ++i) { + const auto &layer = vaFrame.layers[i]; // Max 33 attributes (1 key + 1 value for each) const int EGL_ATTRIB_COUNT = 33 * 2; @@ -264,7 +373,7 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip int attribIndex = 8; for (size_t j = 0; j < layer.num_planes; j++) { - const auto &object = vaFrame->objects[layer.object_index[j]]; + const auto &object = vaFrame.objects[layer.object_index[j]]; switch (j) { case 0: @@ -333,7 +442,7 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip } // For composed exports, add the YUV metadata - if (vaFrame->num_layers == 1) { + if (vaFrame.num_layers == 1) { // Add colorspace metadata switch (m_Renderer->getFrameColorspace(frame)) { case COLORSPACE_REC_601: @@ -392,13 +501,13 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip SDL_assert(attribIndex <= EGL_ATTRIB_COUNT); if (m_eglCreateImage) { - images[i] = m_eglCreateImage(dpy, EGL_NO_CONTEXT, - EGL_LINUX_DMA_BUF_EXT, - nullptr, attribs); - if (!images[i]) { + imgCtx.images[i] = m_eglCreateImage(dpy, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + nullptr, attribs); + if (!imgCtx.images[i]) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "eglCreateImage() Failed: %d", eglGetError()); - goto fail; + break; } } else { @@ -408,24 +517,37 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip intAttribs[i] = (EGLint)attribs[i]; } - images[i] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT, - EGL_LINUX_DMA_BUF_EXT, - nullptr, intAttribs); - if (!images[i]) { + imgCtx.images[i] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + nullptr, intAttribs); + if (!imgCtx.images[i]) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "eglCreateImageKHR() Failed: %d", eglGetError()); - goto fail; + break; } } - ++count; + imgCtx.count++; } - return count; + // Always close the exported FDs + for (size_t i = 0; i < vaFrame.num_objects; ++i) { + close(vaFrame.objects[i].fd); + } -fail: - freeEGLImages(dpy, images); - return -1; + // Check for failure + if (vaFrame.num_layers != imgCtx.count) { + return -1; + } + + // Copy the output from the image context before we move it + ssize_t count = imgCtx.count; + memcpy(images, imgCtx.images, sizeof(EGLImage) * imgCtx.count); + + // Move this image context into the cache + populateImageCache(frame, std::move(imgCtx)); + + return count; } bool EglImageFactory::supportsImportingFormat(EGLDisplay dpy, EGLint format) @@ -512,15 +634,12 @@ bool EglImageFactory::supportsImportingModifier(EGLDisplay dpy, EGLint format, E #endif void EglImageFactory::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { - for (size_t i = 0; i < EGL_MAX_PLANES; ++i) { - if (images[i] != nullptr) { - if (m_eglDestroyImage) { - m_eglDestroyImage(dpy, images[i]); - } - else { - m_eglDestroyImageKHR(dpy, images[i]); - } - } + Q_UNUSED(dpy); + Q_UNUSED(images); + + // When the cache is disabled, we just insert one element at a time + // with key of nullptr and clear it after every frame. + if (m_CacheDisabled) { + m_CachedImages.clear(); } - memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES); } diff --git a/app/streaming/video/ffmpeg-renderers/eglimagefactory.h b/app/streaming/video/ffmpeg-renderers/eglimagefactory.h index e185e9de..818be49a 100644 --- a/app/streaming/video/ffmpeg-renderers/eglimagefactory.h +++ b/app/streaming/video/ffmpeg-renderers/eglimagefactory.h @@ -2,22 +2,66 @@ #include "renderer.h" +#include + #ifdef HAVE_LIBVA #include #endif class EglImageFactory { + class EglImageContext { + public: + EglImageContext(EGLDisplay display, PFNEGLDESTROYIMAGEPROC fn_eglDestroyImage, PFNEGLDESTROYIMAGEKHRPROC fn_eglDestroyImageKHR) : + m_Display(display), + m_eglDestroyImage(fn_eglDestroyImage), + m_eglDestroyImageKHR(fn_eglDestroyImageKHR) {} + + EglImageContext(const EglImageContext& other) = delete; + + EglImageContext(EglImageContext&& other) { + // Copy the basic EGL state + m_Display = other.m_Display; + m_eglDestroyImage = other.m_eglDestroyImage; + m_eglDestroyImageKHR = other.m_eglDestroyImageKHR; + + // Transfer ownership of the EGLImages + memcpy(images, other.images, other.count * sizeof(EGLImage)); + count = other.count; + other.count = 0; + } + + ~EglImageContext() { + for (ssize_t i = 0; i < count; ++i) { + if (m_eglDestroyImage) { + m_eglDestroyImage(m_Display, images[i]); + } + else { + m_eglDestroyImageKHR(m_Display, images[i]); + } + } + } + + EGLImage images[EGL_MAX_PLANES] {}; + ssize_t count = 0; + + private: + EGLDisplay m_Display; + PFNEGLDESTROYIMAGEPROC m_eglDestroyImage; + PFNEGLDESTROYIMAGEKHRPROC m_eglDestroyImageKHR; + }; + public: EglImageFactory(IFFmpegRenderer* renderer); bool initializeEGL(EGLDisplay, const EGLExtensions &ext); + void resetCache(); #ifdef HAVE_DRM - ssize_t exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); + ssize_t exportDRMImages(AVFrame* frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); #endif #ifdef HAVE_LIBVA - ssize_t exportVAImages(AVFrame* frame, VADRMPRIMESurfaceDescriptor* vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); + ssize_t exportVAImages(AVFrame* frame, uint32_t exportFlags, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); #endif bool supportsImportingFormat(EGLDisplay dpy, EGLint format); @@ -25,8 +69,14 @@ public: void freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); +private: + ssize_t queryImageCache(AVFrame* frame, EGLImage images[EGL_MAX_PLANES]); + void populateImageCache(AVFrame* frame, EglImageContext &&imgCtx); + private: IFFmpegRenderer* m_Renderer; + bool m_CacheDisabled; + std::unordered_map m_CachedImages; bool m_EGLExtDmaBuf; PFNEGLCREATEIMAGEPROC m_eglCreateImage; PFNEGLDESTROYIMAGEPROC m_eglDestroyImage; diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.cpp b/app/streaming/video/ffmpeg-renderers/vaapi.cpp index b6d31e8e..72a11954 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.cpp +++ b/app/streaming/video/ffmpeg-renderers/vaapi.cpp @@ -30,10 +30,6 @@ VAAPIRenderer::VAAPIRenderer(int decoderSelectionPass) m_EglImageFactory(this) #endif { -#ifdef HAVE_EGL - SDL_zero(m_PrimeDescriptor); -#endif - #ifdef HAVE_LIBVA_X11 m_XDisplay = nullptr; m_XWindow = None; @@ -574,6 +570,16 @@ VAAPIRenderer::prepareDecoderContext(AVCodecContext* context, AVDictionary**) return true; } +bool +VAAPIRenderer::prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat) +{ +#ifdef HAVE_EGL + // The surface pool is being reset, so clear the cached EGLImages + m_EglImageFactory.resetCache(); +#endif + return true; +} + bool VAAPIRenderer::isDirectRenderingSupported() { @@ -1138,7 +1144,6 @@ VAAPIRenderer::initializeEGL(EGLDisplay dpy, ssize_t VAAPIRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { - ssize_t count; uint32_t exportFlags = VA_EXPORT_SURFACE_READ_ONLY; switch (m_EglExportType) { @@ -1153,52 +1158,12 @@ VAAPIRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy, return -1; } - auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data; - AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)hwFrameCtx->device_ctx->hwctx; - VASurfaceID surface_id = (VASurfaceID)(uintptr_t)frame->data[3]; - - VAStatus st = vaExportSurfaceHandle(vaDeviceContext->display, - surface_id, - VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, - exportFlags, - &m_PrimeDescriptor); - if (st != VA_STATUS_SUCCESS) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "vaExportSurfaceHandle failed: %d", st); - return -1; - } - - st = vaSyncSurface(vaDeviceContext->display, surface_id); - if (st != VA_STATUS_SUCCESS) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "vaSyncSurface() failed: %d", st); - goto fail; - } - - count = m_EglImageFactory.exportVAImages(frame, &m_PrimeDescriptor, dpy, images); - if (count < 0) { - goto fail; - } - - return count; - -fail: - for (size_t i = 0; i < m_PrimeDescriptor.num_objects; ++i) { - close(m_PrimeDescriptor.objects[i].fd); - } - m_PrimeDescriptor.num_layers = 0; - m_PrimeDescriptor.num_objects = 0; - return -1; + return m_EglImageFactory.exportVAImages(frame, exportFlags, dpy, images); } void VAAPIRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { m_EglImageFactory.freeEGLImages(dpy, images); - for (size_t i = 0; i < m_PrimeDescriptor.num_objects; ++i) { - close(m_PrimeDescriptor.objects[i].fd); - } - m_PrimeDescriptor.num_layers = 0; - m_PrimeDescriptor.num_objects = 0; } #endif diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.h b/app/streaming/video/ffmpeg-renderers/vaapi.h index a1705b43..301324ca 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.h +++ b/app/streaming/video/ffmpeg-renderers/vaapi.h @@ -60,6 +60,7 @@ public: virtual ~VAAPIRenderer() override; virtual bool initialize(PDECODER_PARAMETERS params) override; virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override; + virtual bool prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat) override; virtual void renderFrame(AVFrame* frame) override; virtual bool isDirectRenderingSupported() override; virtual int getDecoderColorspace() override; @@ -122,7 +123,6 @@ private: Separate, Composed } m_EglExportType; - VADRMPRIMESurfaceDescriptor m_PrimeDescriptor; EglImageFactory m_EglImageFactory; #endif };