moonlight-stream/moonlight-qt
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt.git synced 2026-02-16 02:30:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add EGLImage caching to improve performance

Browse Source
This commit is contained in:
Cameron Gutman
2025-12-28 17:08:45 -06:00
parent c3ce0918b3
commit 085480e9f4
6 changed files with 236 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
app/streaming/video/ffmpeg-renderers/drm.cpp
View File

@@ -277,6 +277,15 @@ bool DrmRenderer::prepareDecoderContext(AVCodecContext* context, AVDictionary**
return true;
}
bool DrmRenderer::prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat)
{
#ifdef HAVE_EGL
// The surface pool is being reset, so clear the cached EGLImages
m_EglImageFactory.resetCache();
#endif
return true;
}
void DrmRenderer::prepareToRender()
{
// Retake DRM master if we dropped it earlier
@@ -1485,8 +1494,7 @@ ssize_t DrmRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy,
return -1;
}
AVDRMFrameDescriptor* drmFrame = (AVDRMFrameDescriptor*)frame->data[0];
return m_EglImageFactory.exportDRMImages(frame, drmFrame, dpy, images);
return m_EglImageFactory.exportDRMImages(frame, dpy, images);
}
void DrmRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) {

1
app/streaming/video/ffmpeg-renderers/drm.h
View File

@@ -54,6 +54,7 @@ public:
virtual ~DrmRenderer() override;
virtual bool initialize(PDECODER_PARAMETERS params) override;
virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override;
virtual bool prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat) override;
virtual void prepareToRender() override;
virtual void renderFrame(AVFrame* frame) override;
virtual enum AVPixelFormat getPreferredPixelFormat(int videoFormat) override;

203
app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp
View File

@@ -1,5 +1,10 @@
#include "eglimagefactory.h"
#ifdef HAVE_LIBVA
#include <libavutil/hwcontext_vaapi.h>
#include <unistd.h>
#endif
#include <vector>
// Don't take a dependency on libdrm just for these constants
@@ -21,6 +26,7 @@
EglImageFactory::EglImageFactory(IFFmpegRenderer* renderer) :
m_Renderer(renderer),
m_CacheDisabled(false),
m_EGLExtDmaBuf(false),
m_eglCreateImage(nullptr),
m_eglDestroyImage(nullptr),
@@ -62,12 +68,76 @@ bool EglImageFactory::initializeEGL(EGLDisplay,
return true;
}
void EglImageFactory::resetCache()
{
m_CachedImages.clear();
}
ssize_t EglImageFactory::queryImageCache(AVFrame *frame, EGLImage images[EGL_MAX_PLANES])
{
if (m_CacheDisabled) {
return -1;
}
else if (!frame->hw_frames_ctx) {
// If we don't have a AVHWFramesContext, we won't have a frame pool
// which means our caching logic of checking AVBuffer pointers won't
// work properly.
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"EGLImage caching disabled due to missing AVHWFramesContext");
m_CacheDisabled = true;
return -1;
}
else if (!frame->buf[0]) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"EGLImage caching disabled due to missing AVBufferRef");
m_CacheDisabled = true;
return -1;
}
else if (!frame->buf[0]->buffer) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"EGLImage caching disabled due to missing AVBuffer");
m_CacheDisabled = true;
return -1;
}
else if (m_CachedImages.size() >= 20) {
// This is a final fail-safe for the case where the buffers aren't
// actually being reused to avoid the cache size growing forever.
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"EGLImage caching disabled due to excessive cached image count");
m_CacheDisabled = true;
return -1;
}
auto imgCtx = m_CachedImages.find(frame->buf[0]->buffer);
if (imgCtx == m_CachedImages.end()) {
return -1;
}
memcpy(images, imgCtx->second.images, sizeof(EGLImage) * imgCtx->second.count);
return imgCtx->second.count;
}
void EglImageFactory::populateImageCache(AVFrame *frame, EglImageContext &&imgCtx)
{
AVBuffer* cacheKey = m_CacheDisabled ? nullptr : frame->buf[0]->buffer;
// Move this entry into the cache
m_CachedImages.emplace(cacheKey, std::move(imgCtx));
}
#ifdef HAVE_DRM
ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES])
ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES])
{
memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES);
// Check the cache first
if (ssize_t count = queryImageCache(frame, images); count > 0) {
return count;
}
AVDRMFrameDescriptor* drmFrame = (AVDRMFrameDescriptor*)frame->data[0];
// DRM requires composed layers rather than separate layers per plane
SDL_assert(drmFrame->nb_layers == 1);
@@ -207,12 +277,14 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d
attribs[attribIndex++] = EGL_NONE;
SDL_assert(attribIndex <= MAX_ATTRIB_COUNT);
EglImageContext imgCtx(dpy, m_eglDestroyImage, m_eglDestroyImageKHR);
// Our EGLImages are non-planar, so we only populate the first entry
if (m_eglCreateImage) {
images[0] = m_eglCreateImage(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, attribs);
if (!images[0]) {
imgCtx.images[0] = m_eglCreateImage(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, attribs);
if (!imgCtx.images[0]) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglCreateImage() Failed: %d", eglGetError());
return -1;
@@ -225,16 +297,24 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d
intAttribs[i] = (EGLint)attribs[i];
}
images[0] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, intAttribs);
if (!images[0]) {
imgCtx.images[0] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, intAttribs);
if (!imgCtx.images[0]) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglCreateImageKHR() Failed: %d", eglGetError());
return -1;
}
}
imgCtx.count = 1;
// Copy the output from the image context before we move it
images[0] = imgCtx.images[0];
// Move this image context into the cache
populateImageCache(frame, std::move(imgCtx));
return 1;
}
@@ -242,16 +322,45 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d
#ifdef HAVE_LIBVA
ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescriptor *vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES])
ssize_t EglImageFactory::exportVAImages(AVFrame *frame, uint32_t exportFlags, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES])
{
ssize_t count = 0;
memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES);
SDL_assert(vaFrame->num_layers <= EGL_MAX_PLANES);
auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)hwFrameCtx->device_ctx->hwctx;
VASurfaceID surface_id = (VASurfaceID)(uintptr_t)frame->data[3];
for (size_t i = 0; i < vaFrame->num_layers; ++i) {
const auto &layer = vaFrame->layers[i];
// Sync the surface before doing anything. We need to do this even if we've got cached EGLImages.
VAStatus st = vaSyncSurface(vaDeviceContext->display, surface_id);
if (st != VA_STATUS_SUCCESS) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"vaSyncSurface() failed: %d", st);
return -1;
}
// Check the cache first
if (ssize_t count = queryImageCache(frame, images); count > 0) {
return count;
}
EglImageContext imgCtx(dpy, m_eglDestroyImage, m_eglDestroyImageKHR);
VADRMPRIMESurfaceDescriptor vaFrame;
st = vaExportSurfaceHandle(vaDeviceContext->display,
surface_id,
VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
exportFlags,
&vaFrame);
if (st != VA_STATUS_SUCCESS) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"vaExportSurfaceHandle failed: %d", st);
return -1;
}
SDL_assert(vaFrame.num_layers <= EGL_MAX_PLANES);
for (size_t i = 0; i < vaFrame.num_layers; ++i) {
const auto &layer = vaFrame.layers[i];
// Max 33 attributes (1 key + 1 value for each)
const int EGL_ATTRIB_COUNT = 33 * 2;
@@ -264,7 +373,7 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip
int attribIndex = 8;
for (size_t j = 0; j < layer.num_planes; j++) {
const auto &object = vaFrame->objects[layer.object_index[j]];
const auto &object = vaFrame.objects[layer.object_index[j]];
switch (j) {
case 0:
@@ -333,7 +442,7 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip
}
// For composed exports, add the YUV metadata
if (vaFrame->num_layers == 1) {
if (vaFrame.num_layers == 1) {
// Add colorspace metadata
switch (m_Renderer->getFrameColorspace(frame)) {
case COLORSPACE_REC_601:
@@ -392,13 +501,13 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip
SDL_assert(attribIndex <= EGL_ATTRIB_COUNT);
if (m_eglCreateImage) {
images[i] = m_eglCreateImage(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, attribs);
if (!images[i]) {
imgCtx.images[i] = m_eglCreateImage(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, attribs);
if (!imgCtx.images[i]) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglCreateImage() Failed: %d", eglGetError());
goto fail;
break;
}
}
else {
@@ -408,24 +517,37 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip
intAttribs[i] = (EGLint)attribs[i];
}
images[i] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, intAttribs);
if (!images[i]) {
imgCtx.images[i] = m_eglCreateImageKHR(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, intAttribs);
if (!imgCtx.images[i]) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglCreateImageKHR() Failed: %d", eglGetError());
goto fail;
break;
}
}
++count;
imgCtx.count++;
}
return count;
// Always close the exported FDs
for (size_t i = 0; i < vaFrame.num_objects; ++i) {
close(vaFrame.objects[i].fd);
}
fail:
freeEGLImages(dpy, images);
return -1;
// Check for failure
if (vaFrame.num_layers != imgCtx.count) {
return -1;
}
// Copy the output from the image context before we move it
ssize_t count = imgCtx.count;
memcpy(images, imgCtx.images, sizeof(EGLImage) * imgCtx.count);
// Move this image context into the cache
populateImageCache(frame, std::move(imgCtx));
return count;
}
bool EglImageFactory::supportsImportingFormat(EGLDisplay dpy, EGLint format)
@@ -512,15 +634,12 @@ bool EglImageFactory::supportsImportingModifier(EGLDisplay dpy, EGLint format, E
#endif
void EglImageFactory::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) {
for (size_t i = 0; i < EGL_MAX_PLANES; ++i) {
if (images[i] != nullptr) {
if (m_eglDestroyImage) {
m_eglDestroyImage(dpy, images[i]);
}
else {
m_eglDestroyImageKHR(dpy, images[i]);
}
}
Q_UNUSED(dpy);
Q_UNUSED(images);
// When the cache is disabled, we just insert one element at a time
// with key of nullptr and clear it after every frame.
if (m_CacheDisabled) {
m_CachedImages.clear();
}
memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES);
}

54
app/streaming/video/ffmpeg-renderers/eglimagefactory.h
View File

@@ -2,22 +2,66 @@
#include "renderer.h"
#include <unordered_map>
#ifdef HAVE_LIBVA
#include <va/va_drmcommon.h>
#endif
class EglImageFactory
{
class EglImageContext {
public:
EglImageContext(EGLDisplay display, PFNEGLDESTROYIMAGEPROC fn_eglDestroyImage, PFNEGLDESTROYIMAGEKHRPROC fn_eglDestroyImageKHR) :
m_Display(display),
m_eglDestroyImage(fn_eglDestroyImage),
m_eglDestroyImageKHR(fn_eglDestroyImageKHR) {}
EglImageContext(const EglImageContext& other) = delete;
EglImageContext(EglImageContext&& other) {
// Copy the basic EGL state
m_Display = other.m_Display;
m_eglDestroyImage = other.m_eglDestroyImage;
m_eglDestroyImageKHR = other.m_eglDestroyImageKHR;
// Transfer ownership of the EGLImages
memcpy(images, other.images, other.count * sizeof(EGLImage));
count = other.count;
other.count = 0;
}
~EglImageContext() {
for (ssize_t i = 0; i < count; ++i) {
if (m_eglDestroyImage) {
m_eglDestroyImage(m_Display, images[i]);
}
else {
m_eglDestroyImageKHR(m_Display, images[i]);
}
}
}
EGLImage images[EGL_MAX_PLANES] {};
ssize_t count = 0;
private:
EGLDisplay m_Display;
PFNEGLDESTROYIMAGEPROC m_eglDestroyImage;
PFNEGLDESTROYIMAGEKHRPROC m_eglDestroyImageKHR;
};
public:
EglImageFactory(IFFmpegRenderer* renderer);
bool initializeEGL(EGLDisplay, const EGLExtensions &ext);
void resetCache();
#ifdef HAVE_DRM
ssize_t exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
ssize_t exportDRMImages(AVFrame* frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
#endif
#ifdef HAVE_LIBVA
ssize_t exportVAImages(AVFrame* frame, VADRMPRIMESurfaceDescriptor* vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
ssize_t exportVAImages(AVFrame* frame, uint32_t exportFlags, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
#endif
bool supportsImportingFormat(EGLDisplay dpy, EGLint format);
@@ -25,8 +69,14 @@ public:
void freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
private:
ssize_t queryImageCache(AVFrame* frame, EGLImage images[EGL_MAX_PLANES]);
void populateImageCache(AVFrame* frame, EglImageContext &&imgCtx);
private:
IFFmpegRenderer* m_Renderer;
bool m_CacheDisabled;
std::unordered_map<AVBuffer*, EglImageContext> m_CachedImages;
bool m_EGLExtDmaBuf;
PFNEGLCREATEIMAGEPROC m_eglCreateImage;
PFNEGLDESTROYIMAGEPROC m_eglDestroyImage;

57
app/streaming/video/ffmpeg-renderers/vaapi.cpp
View File

@@ -30,10 +30,6 @@ VAAPIRenderer::VAAPIRenderer(int decoderSelectionPass)
m_EglImageFactory(this)
#endif
{
#ifdef HAVE_EGL
SDL_zero(m_PrimeDescriptor);
#endif
#ifdef HAVE_LIBVA_X11
m_XDisplay = nullptr;
m_XWindow = None;
@@ -574,6 +570,16 @@ VAAPIRenderer::prepareDecoderContext(AVCodecContext* context, AVDictionary**)
return true;
}
bool
VAAPIRenderer::prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat)
{
#ifdef HAVE_EGL
// The surface pool is being reset, so clear the cached EGLImages
m_EglImageFactory.resetCache();
#endif
return true;
}
bool
VAAPIRenderer::isDirectRenderingSupported()
{
@@ -1138,7 +1144,6 @@ VAAPIRenderer::initializeEGL(EGLDisplay dpy,
ssize_t
VAAPIRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy,
EGLImage images[EGL_MAX_PLANES]) {
ssize_t count;
uint32_t exportFlags = VA_EXPORT_SURFACE_READ_ONLY;
switch (m_EglExportType) {
@@ -1153,52 +1158,12 @@ VAAPIRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy,
return -1;
}
auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)hwFrameCtx->device_ctx->hwctx;
VASurfaceID surface_id = (VASurfaceID)(uintptr_t)frame->data[3];
VAStatus st = vaExportSurfaceHandle(vaDeviceContext->display,
surface_id,
VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
exportFlags,
&m_PrimeDescriptor);
if (st != VA_STATUS_SUCCESS) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"vaExportSurfaceHandle failed: %d", st);
return -1;
}
st = vaSyncSurface(vaDeviceContext->display, surface_id);
if (st != VA_STATUS_SUCCESS) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"vaSyncSurface() failed: %d", st);
goto fail;
}
count = m_EglImageFactory.exportVAImages(frame, &m_PrimeDescriptor, dpy, images);
if (count < 0) {
goto fail;
}
return count;
fail:
for (size_t i = 0; i < m_PrimeDescriptor.num_objects; ++i) {
close(m_PrimeDescriptor.objects[i].fd);
}
m_PrimeDescriptor.num_layers = 0;
m_PrimeDescriptor.num_objects = 0;
return -1;
return m_EglImageFactory.exportVAImages(frame, exportFlags, dpy, images);
}
void
VAAPIRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) {
m_EglImageFactory.freeEGLImages(dpy, images);
for (size_t i = 0; i < m_PrimeDescriptor.num_objects; ++i) {
close(m_PrimeDescriptor.objects[i].fd);
}
m_PrimeDescriptor.num_layers = 0;
m_PrimeDescriptor.num_objects = 0;
}
#endif

2
app/streaming/video/ffmpeg-renderers/vaapi.h
View File

@@ -60,6 +60,7 @@ public:
virtual ~VAAPIRenderer() override;
virtual bool initialize(PDECODER_PARAMETERS params) override;
virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override;
virtual bool prepareDecoderContextInGetFormat(AVCodecContext*, AVPixelFormat) override;
virtual void renderFrame(AVFrame* frame) override;
virtual bool isDirectRenderingSupported() override;
virtual int getDecoderColorspace() override;
@@ -122,7 +123,6 @@ private:
Separate,
Composed
} m_EglExportType;
VADRMPRIMESurfaceDescriptor m_PrimeDescriptor;
EglImageFactory m_EglImageFactory;
#endif
};