moonlight-stream/moonlight-qt
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt.git synced 2026-02-16 10:40:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduce a new FFMPEG frontend renderer: EGLRenderer

Browse Source 
Right now this renderer works on X11 & Wayland with VAAPI as a backend.

Some rendering latency benchmarks on my `i7-10510U` (with
`intel-media-driver` 20.1.1 which cause a *huge* regression with the
SDL_Renderer):
|             | X11    | Wayland |
| Before      | 6.78ms | 22.50ms |
| EGLRenderer | 0.76ms | 00.77ms |

Signed-off-by: Antoine Damhet <antoine.damhet@lse.epita.fr>
This commit is contained in:
Antoine Damhet
2020-04-13 10:40:28 +02:00
parent 2aa18dc88f
commit 0a396f3112
11 changed files with 761 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
app/app.pro
View File

@@ -90,6 +90,10 @@ unix:!macx {
CONFIG += libva
}
packagesExist(egl) {
CONFIG += egl
}
packagesExist(vdpau) {
CONFIG += libvdpau
}
@@ -264,6 +268,15 @@ libdrm {
SOURCES += streaming/video/ffmpeg-renderers/drm.cpp
HEADERS += streaming/video/ffmpeg-renderers/drm.h
}
egl {
message(EGL renderer selected)
DEFINES += HAVE_EGL
SOURCES += \
streaming/video/ffmpeg-renderers/eglvid.cpp \
streaming/video/ffmpeg-renderers/egl_extensions.cpp
HEADERS += streaming/video/ffmpeg-renderers/eglvid.h
}
config_SL {
message(Steam Link build configuration selected)

2
app/resources.qrc
View File

@@ -19,5 +19,7 @@
<qresource prefix="/data">
<file alias="gamecontrollerdb.txt">SDL_GameControllerDB/gamecontrollerdb.txt</file>
<file alias="ModeSeven.ttf">ModeSeven.ttf</file>
<file alias="egl.frag">shaders/egl.frag</file>
<file alias="egl.vert">shaders/egl.vert</file>
</qresource>
</RCC>

21
app/shaders/egl.frag Normal file
View File

@@ -0,0 +1,21 @@
#version 300 es
#extension GL_OES_EGL_image_external : require
precision mediump float;
out vec4 FragColor;
in vec2 vTextCoord;
uniform mat3 yuvmat;
uniform vec3 offset;
uniform samplerExternalOES plane1;
uniform samplerExternalOES plane2;
void main() {
vec3 YCbCr = vec3(
texture2D(plane1, vTextCoord)[0],
texture2D(plane2, vTextCoord).xy
);
YCbCr -= offset;
FragColor = vec4(clamp(yuvmat * YCbCr, 0.0, 1.0), 1.0f);
}

10
app/shaders/egl.vert Normal file
View File

@@ -0,0 +1,10 @@
#version 300 es
layout (location = 0) in vec2 aPosition; // 2D: X,Y
layout (location = 1) in vec2 aTexCoord;
out vec2 vTextCoord;
void main() {
vTextCoord = aTexCoord;
gl_Position = vec4(aPosition, 0, 1);
}

23
app/streaming/video/ffmpeg-renderers/egl_extensions.cpp Normal file
View File

@@ -0,0 +1,23 @@
// vim: noai:ts=4:sw=4:softtabstop=4:expandtab
#define GL_GLEXT_PROTOTYPES
#include "renderer.h"
#include <SDL_egl.h>
static QStringList egl_get_extensions(EGLDisplay dpy) {
const auto EGLExtensionsStr = eglQueryString(dpy, EGL_EXTENSIONS);
if (!EGLExtensionsStr) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Unable to get EGL extensions");
return QStringList();
}
return QString(EGLExtensionsStr).split(" ");
}
EGLExtensions::EGLExtensions(EGLDisplay dpy) :
m_Extensions(egl_get_extensions(dpy))
{}
bool EGLExtensions::isSupported(const QString &extension) const {
return m_Extensions.contains(extension);
}

480
app/streaming/video/ffmpeg-renderers/eglvid.cpp Normal file
View File

@@ -0,0 +1,480 @@
// vim: noai:ts=4:sw=4:softtabstop=4:expandtab
#define GL_GLEXT_PROTOTYPES
#include "eglvid.h"
#include "path.h"
#include "streaming/session.h"
#include "streaming/streamutils.h"
#include <QDir>
#include <Limelight.h>
#include <unistd.h>
#include <SDL_egl.h>
#include <SDL_opengl.h>
#include <SDL_opengles2.h>
#include <SDL_opengles2_gl2ext.h>
#include <SDL_render.h>
#include <SDL_syswm.h>
/* TODO:
* - handle more pixel formats
* - handle software decoding
*/
/* DOC/misc:
* - https://kernel-recipes.org/en/2016/talks/video-and-colorspaces/
* - http://www.brucelindbloom.com/
* - https://learnopengl.com/Getting-started/Shaders
* - https://github.com/stunpix/yuvit
* - https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
* - https://www.renesas.com/eu/en/www/doc/application-note/an9717.pdf
* - https://www.xilinx.com/support/documentation/application_notes/xapp283.pdf
* - https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.709-6-201506-I!!PDF-E.pdf
* - https://www.khronos.org/registry/OpenGL/extensions/OES/OES_EGL_image_external.txt
* - https://gist.github.com/rexguo/6696123
* - https://wiki.libsdl.org/CategoryVideo
*/
#define EGL_LOG(Category, ...) SDL_Log ## Category(\
SDL_LOG_CATEGORY_APPLICATION, \
"EGLRenderer: " __VA_ARGS__)
EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer)
:
m_SwPixelFormat(AV_PIX_FMT_NONE),
m_EGLDisplay(nullptr),
m_Textures{0},
m_ShaderProgram(0),
m_Context(0),
m_Window(nullptr),
m_Backend(backendRenderer),
m_VAO(0),
m_ColorSpace(AVCOL_SPC_NB),
m_ColorFull(false),
EGLImageTargetTexture2DOES(nullptr),
m_DummyRenderer(nullptr)
{
SDL_assert(backendRenderer);
SDL_assert(backendRenderer->canExportEGL());
}
EGLRenderer::~EGLRenderer()
{
if (m_Context) {
if (m_ShaderProgram)
glDeleteProgram(m_ShaderProgram);
if (m_VAO)
glDeleteVertexArrays(1, &m_VAO);
if (m_DummyRenderer)
SDL_DestroyRenderer(m_DummyRenderer);
SDL_GL_DeleteContext(m_Context);
}
}
bool EGLRenderer::prepareDecoderContext(AVCodecContext*, AVDictionary**)
{
/* Nothing to do */
EGL_LOG(Info, "Using EGL renderer");
return true;
}
void EGLRenderer::notifyOverlayUpdated(Overlay::OverlayType)
{
// TODO: FIXME
}
bool EGLRenderer::isRenderThreadSupported()
{
// TODO: can we use DMA-BUF in multithreaded context ?
return false;
}
bool EGLRenderer::isPixelFormatSupported(int, AVPixelFormat pixelFormat)
{
// Remember to keep this in sync with EGLRenderer::renderFrame()!
switch (pixelFormat)
{
case AV_PIX_FMT_NV12:
return true;
default:
return false;
}
}
int EGLRenderer::loadAndBuildShader(int shaderType,
const char *file) {
GLuint shader = glCreateShader(shaderType);
if (!shader || shader == GL_INVALID_ENUM) {
EGL_LOG(Error, "Can't create shader: %d", glGetError());
return 0;
}
auto sourceData = Path::readDataFile(file);
GLint len = sourceData.size();
const char *buf = sourceData.data();
glShaderSource(shader, 1, &buf, &len);
glCompileShader(shader);
GLint status;
glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
if (!status) {
char shaderLog[512];
glGetShaderInfoLog(shader, sizeof (shaderLog), nullptr, shaderLog);
EGL_LOG(Error, "Cannot load shader \"%s\": %s", file, shaderLog);
return 0;
}
return shader;
}
bool EGLRenderer::compileShader() {
SDL_assert(!m_ShaderProgram);
SDL_assert(m_SwPixelFormat != AV_PIX_FMT_NONE);
// XXX: TODO: other formats
SDL_assert(m_SwPixelFormat == AV_PIX_FMT_NV12);
bool ret = false;
GLuint vertexShader = loadAndBuildShader(GL_VERTEX_SHADER, "egl.vert");
if (!vertexShader)
return false;
GLuint fragmentShader = loadAndBuildShader(GL_FRAGMENT_SHADER, "egl.frag");
if (!fragmentShader)
goto fragError;
m_ShaderProgram = glCreateProgram();
if (!m_ShaderProgram) {
EGL_LOG(Error, "Cannot create shader program");
goto progFailCreate;
}
glAttachShader(m_ShaderProgram, vertexShader);
glAttachShader(m_ShaderProgram, fragmentShader);
glLinkProgram(m_ShaderProgram);
int status;
glGetProgramiv(m_ShaderProgram, GL_LINK_STATUS, &status);
if (status) {
ret = true;
} else {
char shader_log[512];
glGetProgramInfoLog(m_ShaderProgram, sizeof (shader_log), nullptr, shader_log);
EGL_LOG(Error, "Cannot link shader program: %s", shader_log);
glDeleteProgram(m_ShaderProgram);
m_ShaderProgram = 0;
}
progFailCreate:
glDeleteShader(fragmentShader);
fragError:
glDeleteShader(vertexShader);
return ret;
}
bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
{
m_Window = params->window;
if (params->videoFormat == VIDEO_FORMAT_H265_MAIN10) {
// EGL doesn't support rendering YUV 10-bit textures yet
return false;
}
/*
* Create a dummy renderer in order to craft an accelerated SDL Window.
* Request opengl ES 3.0 context, otherwise it will SIGSEGV
* https://gitlab.freedesktop.org/mesa/mesa/issues/1011
*/
SDL_SetHint(SDL_HINT_OPENGL_ES_DRIVER, "1");
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
int renderIndex;
int maxRenderers = SDL_GetNumRenderDrivers();
SDL_assert(maxRenderers >= 0);
SDL_RendererInfo renderInfo;
for (renderIndex = 0; renderIndex < maxRenderers; ++renderIndex) {
if (SDL_GetRenderDriverInfo(renderIndex, &renderInfo))
continue;
if (!strcmp(renderInfo.name, "opengles2")) {
SDL_assert(renderInfo.flags & SDL_RENDERER_ACCELERATED);
break;
}
}
if (renderIndex == maxRenderers) {
EGL_LOG(Error, "Could not find a suitable SDL_Renderer");
return false;
}
if (!(m_DummyRenderer = SDL_CreateRenderer(m_Window, renderIndex, SDL_RENDERER_ACCELERATED))) {
EGL_LOG(Error, "SDL_CreateRenderer() failed: %s", SDL_GetError());
return false;
}
SDL_SysWMinfo info;
SDL_VERSION(&info.version);
if (!SDL_GetWindowWMInfo(params->window, &info)) {
EGL_LOG(Error, "SDL_GetWindowWMInfo() failed: %s", SDL_GetError());
return false;
}
switch (info.subsystem) {
case SDL_SYSWM_WAYLAND:
m_EGLDisplay = eglGetPlatformDisplay(EGL_PLATFORM_WAYLAND_KHR,
info.info.wl.display, nullptr);
break;
case SDL_SYSWM_X11:
m_EGLDisplay = eglGetPlatformDisplay(EGL_PLATFORM_X11_KHR,
info.info.x11.display, nullptr);
break;
default:
EGL_LOG(Error, "not compatible with SYSWM");
return false;
}
if (!m_EGLDisplay) {
EGL_LOG(Error, "Cannot get EGL display: %d", eglGetError());
return false;
}
if (!(m_Context = SDL_GL_CreateContext(params->window))) {
EGL_LOG(Error, "Cannot create OpenGL context: %s", SDL_GetError());
return false;
}
if (SDL_GL_MakeCurrent(params->window, m_Context)) {
EGL_LOG(Error, "Cannot use created EGL context: %s", SDL_GetError());
return false;
}
const EGLExtensions egl_extensions(m_EGLDisplay);
if (!egl_extensions.isSupported("EGL_KHR_image_base") &&
!egl_extensions.isSupported("EGL_KHR_image")) {
EGL_LOG(Error, "KHR_image unsupported");
return false;
}
if (!m_Backend->initializeEGL(m_EGLDisplay, egl_extensions))
return false;
if (!(EGLImageTargetTexture2DOES = (EGLImageTargetTexture2DOES_t)eglGetProcAddress("glEGLImageTargetTexture2DOES"))) {
EGL_LOG(Error,
"EGL: cannot retrieve `EGLImageTargetTexture2DOES` address");
return false;
}
/* Compute the video region size in order to keep the aspect ratio of the
* video stream.
*/
SDL_Rect src, dst;
src.x = src.y = dst.x = dst.y = 0;
src.w = params->width;
src.h = params->height;
SDL_GetWindowSize(m_Window, &dst.w, &dst.h);
StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
glViewport(dst.x, dst.y, dst.w, dst.h);
glClearColor(0, 0, 0, 1);
glClear(GL_COLOR_BUFFER_BIT);
if (params->enableVsync) {
/* Try to use adaptive VSYNC */
if (SDL_GL_SetSwapInterval(-1))
SDL_GL_SetSwapInterval(1);
} else {
SDL_GL_SetSwapInterval(0);
}
SDL_GL_SwapWindow(params->window);
glGenTextures(EGL_MAX_PLANES, m_Textures);
for (size_t i = 0; i < EGL_MAX_PLANES; ++i) {
glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
GLenum err = glGetError();
if (err != GL_NO_ERROR)
EGL_LOG(Error, "OpenGL error: %d", err);
return err == GL_NO_ERROR;
}
const float *EGLRenderer::getColorMatrix() {
/* The conversion matrices are shamelessly stolen from linux:
* drivers/media/platform/imx-pxp.c:pxp_setup_csc
*/
static const float bt601Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.3917f, 2.0172f,
1.5960f, -0.8129f, 0.0f
};
static const float bt601Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.3441f, 1.7720f,
1.4020f, -0.7141f, 0.0f
};
static const float bt709Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.2132f, 2.1124f,
1.7927f, -0.5329f, 0.0f
};
static const float bt709Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.1873f, 1.8556f,
1.5748f, -0.4681f, 0.0f
};
static const float bt2020Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.1874f, 2.1418f,
1.6781f, -0.6505f, 0.0f
};
static const float bt2020Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.1646f, 1.8814f,
1.4746f, -0.5714f, 0.0f
};
switch (m_ColorSpace) {
case AVCOL_SPC_SMPTE170M:
case AVCOL_SPC_BT470BG:
EGL_LOG(Info, "BT-601 pixels");
return m_ColorFull ? bt601Full : bt601Lim;
case AVCOL_SPC_BT709:
EGL_LOG(Info, "BT-709 pixels");
return m_ColorFull ? bt709Full : bt709Lim;
case AVCOL_SPC_BT2020_NCL:
case AVCOL_SPC_BT2020_CL:
EGL_LOG(Info, "BT-2020 pixels");
return m_ColorFull ? bt2020Full : bt2020Lim;
};
EGL_LOG(Warn, "unknown color space: %d, falling back to BT-601",
m_ColorSpace);
return bt601Lim;
}
bool EGLRenderer::specialize() {
SDL_assert(!m_VAO);
if (!compileShader())
return false;
// The viewport should have the aspect ratio of the video stream
static const float vertices[] = {
// pos .... // tex coords
1.0f, 1.0f, 1.0f, 0.0f,
1.0f, -1.0f, 1.0f, 1.0f,
-1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 0.0f,
};
static const unsigned int indices[] = {
0, 1, 3,
1, 2, 3,
};
glUseProgram(m_ShaderProgram);
unsigned int VBO, EBO;
glGenVertexArrays(1, &m_VAO);
glGenBuffers(1, &VBO);
glGenBuffers(1, &EBO);
glBindVertexArray(m_VAO);
glBindBuffer(GL_ARRAY_BUFFER, VBO);
glBufferData(GL_ARRAY_BUFFER, sizeof (vertices), vertices, GL_STATIC_DRAW);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, EBO);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, GL_STATIC_DRAW);
glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
glEnableVertexAttribArray(0);
glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof (float)));
glEnableVertexAttribArray(1);
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
int yuvmatLocation = glGetUniformLocation(m_ShaderProgram, "yuvmat");
glUniformMatrix3fv(yuvmatLocation, 1, GL_FALSE, getColorMatrix());
static const float limitedOffsets[] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f };
static const float fullOffsets[] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f };
int offLocation = glGetUniformLocation(m_ShaderProgram, "offset");
glUniform3fv(offLocation, 1, m_ColorFull ? fullOffsets : limitedOffsets);
int colorPlane = glGetUniformLocation(m_ShaderProgram, "plane1");
glUniform1i(colorPlane, 0);
colorPlane = glGetUniformLocation(m_ShaderProgram, "plane2");
glUniform1i(colorPlane, 1);
glDeleteBuffers(1, &VBO);
glDeleteBuffers(1, &EBO);
GLenum err = glGetError();
if (err != GL_NO_ERROR) {
EGL_LOG(Error, "OpenGL error: %d", err);
}
return err == GL_NO_ERROR;
}
void EGLRenderer::renderFrame(AVFrame* frame)
{
EGLImage imgs[EGL_MAX_PLANES];
if (frame->hw_frames_ctx != nullptr) {
// Find the native read-back format and load the shader
if (m_SwPixelFormat == AV_PIX_FMT_NONE) {
auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
m_SwPixelFormat = hwFrameCtx->sw_format;
SDL_assert(m_SwPixelFormat != AV_PIX_FMT_NONE);
EGL_LOG(Info, "Selected read-back format: %d", m_SwPixelFormat);
// XXX: TODO: Handle other pixel formats
SDL_assert(m_SwPixelFormat == AV_PIX_FMT_NV12);
m_ColorSpace = frame->colorspace;
m_ColorFull = frame->color_range == AVCOL_RANGE_JPEG;
if (!specialize()) {
m_SwPixelFormat = AV_PIX_FMT_NONE;
return;
}
}
ssize_t plane_count = m_Backend->exportEGLImages(frame, m_EGLDisplay, imgs);
if (plane_count < 0)
return;
for (ssize_t i = 0; i < plane_count; ++i) {
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]);
EGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, imgs[i]);
}
} else {
// TODO: load texture for SW decoding ?
EGL_LOG(Error, "EGL rendering only supports hw frames");
return;
}
glClear(GL_COLOR_BUFFER_BIT);
glUseProgram(m_ShaderProgram);
glBindVertexArray(m_VAO);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
SDL_GL_SwapWindow(m_Window);
if (frame->hw_frames_ctx != nullptr)
m_Backend->freeEGLImages(m_EGLDisplay, imgs);
}

36
app/streaming/video/ffmpeg-renderers/eglvid.h Normal file
View File

@@ -0,0 +1,36 @@
#pragma once
#include "renderer.h"
class EGLRenderer : public IFFmpegRenderer {
public:
EGLRenderer(IFFmpegRenderer *backendRenderer);
virtual ~EGLRenderer() override;
virtual bool initialize(PDECODER_PARAMETERS params) override;
virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override;
virtual void renderFrame(AVFrame* frame) override;
virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
virtual bool isRenderThreadSupported() override;
virtual bool isPixelFormatSupported(int videoFormat, enum AVPixelFormat pixelFormat) override;
private:
using EGLImageTargetTexture2DOES_t = void (*)(int, void *);
bool compileShader();
bool specialize();
const float *getColorMatrix();
static int loadAndBuildShader(int shaderType, const char *filename);
int m_SwPixelFormat;
void *m_EGLDisplay;
unsigned m_Textures[EGL_MAX_PLANES];
unsigned m_ShaderProgram;
SDL_GLContext m_Context;
SDL_Window *m_Window;
IFFmpegRenderer *m_Backend;
unsigned int m_VAO;
int m_ColorSpace;
bool m_ColorFull;
EGLImageTargetTexture2DOES_t EGLImageTargetTexture2DOES;
SDL_Renderer *m_DummyRenderer;
};

38
app/streaming/video/ffmpeg-renderers/renderer.h
View File

@@ -9,6 +9,23 @@ extern "C" {
#include <libavcodec/avcodec.h>
}
#ifdef HAVE_EGL
// SDL_egl.h have too many conflicts, we will do without
typedef void *EGLDisplay;
typedef void *EGLImage;
#define EGL_MAX_PLANES 4
class EGLExtensions {
public:
EGLExtensions(EGLDisplay dpy);
~EGLExtensions() {}
bool isSupported(const QString &extension) const;
private:
const QStringList m_Extensions;
};
#endif
#define RENDERER_ATTRIBUTE_FULLSCREEN_ONLY 0x01
#define RENDERER_ATTRIBUTE_1080P_MAX 0x02
@@ -68,4 +85,25 @@ public:
virtual void notifyOverlayUpdated(Overlay::OverlayType) override {
// Nothing
}
#ifdef HAVE_EGL
// By default we can't do EGL
virtual bool canExportEGL() {
return false;
}
virtual bool initializeEGL(EGLDisplay,
const EGLExtensions &) {
return false;
}
virtual ssize_t exportEGLImages(AVFrame *,
EGLDisplay,
EGLImage[EGL_MAX_PLANES]) {
return -1;
}
// Free the ressources allocated during the last `exportEGLImages` call
virtual void freeEGLImages(EGLDisplay, EGLImage[EGL_MAX_PLANES]) {}
#endif
};

112
app/streaming/video/ffmpeg-renderers/vaapi.cpp
View File

@@ -5,6 +5,9 @@
#include <streaming/streamutils.h>
#include <SDL_syswm.h>
#ifdef HAVE_EGL
#include <SDL_egl.h>
#endif
#include <unistd.h>
#include <fcntl.h>
@@ -14,7 +17,11 @@ VAAPIRenderer::VAAPIRenderer()
m_DrmFd(-1),
m_BlacklistedForDirectRendering(false)
{
#ifdef HAVE_EGL
m_PrimeDescriptor.num_layers = 0;
m_PrimeDescriptor.num_objects = 0;
m_EGLExtDmaBuf = false;
#endif
}
VAAPIRenderer::~VAAPIRenderer()
@@ -442,3 +449,106 @@ VAAPIRenderer::renderFrame(AVFrame* frame)
SDL_assert(false);
}
}
#ifdef HAVE_EGL
bool
VAAPIRenderer::canExportEGL() {
return true;
}
bool
VAAPIRenderer::initializeEGL(EGLDisplay,
const EGLExtensions &ext) {
if (!ext.isSupported("EGL_EXT_image_dma_buf_import")) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"VAAPI-EGL: DMABUF unsupported");
return false;
}
m_EGLExtDmaBuf = ext.isSupported("EGL_EXT_image_dma_buf_import_modifiers");
return true;
}
ssize_t
VAAPIRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy,
EGLImage images[EGL_MAX_PLANES]) {
ssize_t count = 0;
auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)hwFrameCtx->device_ctx->hwctx;
VASurfaceID surface_id = (VASurfaceID)(uintptr_t)frame->data[3];
VAStatus st = vaExportSurfaceHandle(vaDeviceContext->display,
surface_id,
VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS,
&m_PrimeDescriptor);
if (st != VA_STATUS_SUCCESS) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"vaExportSurfaceHandle failed: %d", st);
return -1;
}
SDL_assert(m_PrimeDescriptor.num_layers <= EGL_MAX_PLANES);
st = vaSyncSurface(vaDeviceContext->display, surface_id);
if (st != VA_STATUS_SUCCESS) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"vaSyncSurface failed: %d", st);
goto sync_fail;
}
for (size_t i = 0; i < m_PrimeDescriptor.num_layers; ++i) {
const auto &layer = m_PrimeDescriptor.layers[i];
const auto &object = m_PrimeDescriptor.objects[layer.object_index[0]];
EGLAttrib attribs[17] = {
EGL_LINUX_DRM_FOURCC_EXT, (EGLint)layer.drm_format,
EGL_WIDTH, i == 0 ? frame->width : frame->width / 2,
EGL_HEIGHT, i == 0 ? frame->height : frame->height / 2,
EGL_DMA_BUF_PLANE0_FD_EXT, object.fd,
EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLint)layer.offset[0],
EGL_DMA_BUF_PLANE0_PITCH_EXT, (EGLint)layer.pitch[0],
EGL_NONE,
};
if (m_EGLExtDmaBuf) {
const EGLAttrib extra[] = {
EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT,
(EGLint)object.drm_format_modifier,
EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT,
(EGLint)(object.drm_format_modifier >> 32),
EGL_NONE,
};
memcpy((void *)(&attribs[12]), (void *)extra, sizeof (extra));
}
images[i] = eglCreateImage(dpy, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT,
nullptr, attribs);
if (!images[i]) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglCreateImage() Failed: %d", eglGetError());
goto create_image_fail;
}
++count;
}
return count;
create_image_fail:
m_PrimeDescriptor.num_layers = count;
sync_fail:
freeEGLImages(dpy, images);
return -1;
}
void
VAAPIRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) {
for (size_t i = 0; i < m_PrimeDescriptor.num_layers; ++i) {
eglDestroyImage(dpy, images[i]);
}
for (size_t i = 0; i < m_PrimeDescriptor.num_objects; ++i) {
close(m_PrimeDescriptor.objects[i].fd);
}
m_PrimeDescriptor.num_layers = 0;
m_PrimeDescriptor.num_objects = 0;
}
#endif

14
app/streaming/video/ffmpeg-renderers/vaapi.h
View File

@@ -26,6 +26,9 @@ extern "C" {
#include <va/va_drm.h>
#endif
#include <libavutil/hwcontext_vaapi.h>
#ifdef HAVE_EGL
#include <va/va_drmcommon.h>
#endif
}
class VAAPIRenderer : public IFFmpegRenderer
@@ -39,6 +42,12 @@ public:
virtual bool needsTestFrame() override;
virtual bool isDirectRenderingSupported() override;
virtual int getDecoderColorspace() override;
#ifdef HAVE_EGL
virtual bool canExportEGL() override;
virtual bool initializeEGL(EGLDisplay dpy, const EGLExtensions &ext);
virtual ssize_t exportEGLImages(AVFrame *frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) override;
virtual void freeEGLImages(EGLDisplay dpy, EGLImage[EGL_MAX_PLANES]) override;
#endif
private:
bool validateDriver(VADisplay display);
@@ -57,4 +66,9 @@ private:
int m_VideoHeight;
int m_DisplayWidth;
int m_DisplayHeight;
#ifdef HAVE_EGL
VADRMPRIMESurfaceDescriptor m_PrimeDescriptor;
bool m_EGLExtDmaBuf;
#endif
};

13
app/streaming/video/ffmpeg.cpp
View File

@@ -32,6 +32,10 @@
#include "ffmpeg-renderers/drm.h"
#endif
#ifdef HAVE_EGL
#include "ffmpeg-renderers/eglvid.h"
#endif
// This is gross but it allows us to use sizeof()
#include "ffmpeg_videosamples.cpp"
@@ -195,6 +199,15 @@ bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params)
m_FrontendRenderer = m_BackendRenderer;
}
else {
#ifdef HAVE_EGL
if (m_BackendRenderer->canExportEGL()) {
m_FrontendRenderer = new EGLRenderer(m_BackendRenderer);
if (m_FrontendRenderer->initialize(params)) {
return true;
}
delete m_FrontendRenderer;
}
#endif
// The backend renderer cannot directly render to the display, so
// we will create an SDL renderer to draw the frames.
m_FrontendRenderer = new SdlRenderer();