From 481f23b6e9a5568d0a59e8773df9c3fca2dcbde0 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sun, 17 Dec 2023 20:39:26 -0600 Subject: [PATCH] Remove some dependencies on fixed window and frame sizes --- .../video/ffmpeg-renderers/eglvid.cpp | 39 ++++++++++--------- app/streaming/video/ffmpeg-renderers/eglvid.h | 5 +-- .../video/ffmpeg-renderers/sdlvid.cpp | 38 +++++++++++------- .../video/ffmpeg-renderers/vaapi.cpp | 38 +++++++++++------- app/streaming/video/ffmpeg-renderers/vaapi.h | 5 +-- 5 files changed, 71 insertions(+), 54 deletions(-) diff --git a/app/streaming/video/ffmpeg-renderers/eglvid.cpp b/app/streaming/video/ffmpeg-renderers/eglvid.cpp index db71d170..605eab06 100644 --- a/app/streaming/video/ffmpeg-renderers/eglvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/eglvid.cpp @@ -180,7 +180,7 @@ AVPixelFormat EGLRenderer::getPreferredPixelFormat(int videoFormat) return m_Backend->getPreferredPixelFormat(videoFormat); } -void EGLRenderer::renderOverlay(Overlay::OverlayType type) +void EGLRenderer::renderOverlay(Overlay::OverlayType type, int viewportWidth, int viewportHeight) { // Do nothing if this overlay is disabled if (!Session::get()->getOverlayManager().isOverlayEnabled(type)) { @@ -234,7 +234,7 @@ void EGLRenderer::renderOverlay(Overlay::OverlayType type) else if (type == Overlay::OverlayDebug) { // Top left overlayRect.x = 0; - overlayRect.y = m_ViewportHeight - newSurface->h; + overlayRect.y = viewportHeight - newSurface->h; } else { SDL_assert(false); } @@ -245,7 +245,7 @@ void EGLRenderer::renderOverlay(Overlay::OverlayType type) SDL_FreeSurface(newSurface); // Convert screen space to normalized device coordinates - StreamUtils::screenSpaceToNormalizedDeviceCoords(&overlayRect, m_ViewportWidth, m_ViewportHeight); + StreamUtils::screenSpaceToNormalizedDeviceCoords(&overlayRect, viewportWidth, viewportHeight); OVERLAY_VERTEX verts[] = { @@ -653,21 +653,6 @@ bool EGLRenderer::initialize(PDECODER_PARAMETERS params) m_eglClientWaitSync = nullptr; } - /* Compute the video region size in order to keep the aspect ratio of the - * video stream. - */ - SDL_Rect src, dst; - src.x = src.y = dst.x = dst.y = 0; - src.w = params->width; - src.h = params->height; - SDL_GL_GetDrawableSize(m_Window, &dst.w, &dst.h); - StreamUtils::scaleSourceToDestinationSurface(&src, &dst); - - glViewport(dst.x, dst.y, dst.w, dst.h); - - m_ViewportWidth = dst.w; - m_ViewportHeight = dst.h; - // SDL always uses swap interval 0 under the hood on Wayland systems, // because the compositor guarantees tear-free rendering. In this // situation, swap interval > 0 behaves as a frame pacing option @@ -931,6 +916,20 @@ void EGLRenderer::renderFrame(AVFrame* frame) } glClear(GL_COLOR_BUFFER_BIT); + + int drawableWidth, drawableHeight; + SDL_GL_GetDrawableSize(m_Window, &drawableWidth, &drawableHeight); + + // Set the viewport to the size of the aspect-ratio-scaled video + SDL_Rect src, dst; + src.x = src.y = dst.x = dst.y = 0; + src.w = frame->width; + src.h = frame->height; + dst.w = drawableWidth; + dst.h = drawableHeight; + StreamUtils::scaleSourceToDestinationSurface(&src, &dst); + glViewport(dst.x, dst.y, dst.w, dst.h); + glUseProgram(m_ShaderProgram); m_glBindVertexArrayOES(m_VAO); @@ -949,8 +948,10 @@ void EGLRenderer::renderFrame(AVFrame* frame) m_glBindVertexArrayOES(0); + // Adjust the viewport to the whole window before rendering the overlays + glViewport(0, 0, drawableWidth, drawableHeight); for (int i = 0; i < Overlay::OverlayMax; i++) { - renderOverlay((Overlay::OverlayType)i); + renderOverlay((Overlay::OverlayType)i, drawableWidth, drawableHeight); } SDL_GL_SwapWindow(m_Window); diff --git a/app/streaming/video/ffmpeg-renderers/eglvid.h b/app/streaming/video/ffmpeg-renderers/eglvid.h index a06764bb..6fc66473 100644 --- a/app/streaming/video/ffmpeg-renderers/eglvid.h +++ b/app/streaming/video/ffmpeg-renderers/eglvid.h @@ -22,7 +22,7 @@ public: private: - void renderOverlay(Overlay::OverlayType type); + void renderOverlay(Overlay::OverlayType type, int viewportWidth, int viewportHeight); unsigned compileShader(const char* vertexShaderSrc, const char* fragmentShaderSrc); bool compileShaders(); bool specialize(); @@ -31,9 +31,6 @@ private: static int loadAndBuildShader(int shaderType, const char *filename); bool openDisplay(unsigned int platform, void* nativeDisplay); - int m_ViewportWidth; - int m_ViewportHeight; - AVPixelFormat m_EGLImagePixelFormat; void *m_EGLDisplay; unsigned m_Textures[EGL_MAX_PLANES]; diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp index 75294e3f..277102ef 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp @@ -165,19 +165,6 @@ bool SdlRenderer::initialize(PDECODER_PARAMETERS params) SDL_FlushEvent(SDL_WINDOWEVENT); } - // Calculate the video region size, scaling to fill the output size while - // preserving the aspect ratio of the video stream. - SDL_Rect src, dst; - src.x = src.y = 0; - src.w = params->width; - src.h = params->height; - dst.x = dst.y = 0; - SDL_GetRendererOutputSize(m_Renderer, &dst.w, &dst.h); - StreamUtils::scaleSourceToDestinationSurface(&src, &dst); - - // Ensure the viewport is set to the desired video region - SDL_RenderSetViewport(m_Renderer, &dst); - if (!params->testOnly) { // Draw a black frame until the video stream starts rendering SDL_SetRenderDrawColor(m_Renderer, 0, 0, 0, SDL_ALPHA_OPAQUE); @@ -276,6 +263,15 @@ ReadbackRetry: m_ColorSpace = colorspace; } + // Recreate the texture if the frame changed in size + if (m_Texture != nullptr) { + int width, height; + if (SDL_QueryTexture(m_Texture, nullptr, nullptr, &width, &height) == 0 && (frame->width != width || frame->height != height)) { + SDL_DestroyTexture(m_Texture); + m_Texture = nullptr; + } + } + if (m_Texture == nullptr) { Uint32 sdlFormat; @@ -427,9 +423,25 @@ ReadbackRetry: SDL_RenderClear(m_Renderer); + // Calculate the video region size, scaling to fill the output size while + // preserving the aspect ratio of the video stream. + SDL_Rect src, dst; + src.x = src.y = 0; + src.w = frame->width; + src.h = frame->height; + dst.x = dst.y = 0; + SDL_GetRendererOutputSize(m_Renderer, &dst.w, &dst.h); + StreamUtils::scaleSourceToDestinationSurface(&src, &dst); + + // Ensure the viewport is set to the desired video region + SDL_RenderSetViewport(m_Renderer, &dst); + // Draw the video content itself SDL_RenderCopy(m_Renderer, m_Texture, nullptr, nullptr); + // Reset the viewport to the full window for overlay rendering + SDL_RenderSetViewport(m_Renderer, nullptr); + // Draw the overlays for (int i = 0; i < Overlay::OverlayMax; i++) { renderOverlay((Overlay::OverlayType)i); diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.cpp b/app/streaming/video/ffmpeg-renderers/vaapi.cpp index 82448cf8..52d1b7c6 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.cpp +++ b/app/streaming/video/ffmpeg-renderers/vaapi.cpp @@ -206,11 +206,8 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params) { int err; + m_Window = params->window; m_VideoFormat = params->videoFormat; - m_VideoWidth = params->width; - m_VideoHeight = params->height; - - SDL_GetWindowSize(params->window, &m_DisplayWidth, &m_DisplayHeight); m_HwContext = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI); if (!m_HwContext) { @@ -652,7 +649,7 @@ void VAAPIRenderer::notifyOverlayUpdated(Overlay::OverlayType type) if (type == Overlay::OverlayStatusUpdate) { // Bottom Left overlayRect.x = 0; - overlayRect.y = m_DisplayHeight - newSurface->h; + overlayRect.y = -newSurface->h; } else if (type == Overlay::OverlayDebug) { // Top left @@ -691,13 +688,16 @@ VAAPIRenderer::renderFrame(AVFrame* frame) AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data; AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx; + int windowWidth, windowHeight; + SDL_GetWindowSize(m_Window, &windowWidth, &windowHeight); + SDL_Rect src, dst; src.x = src.y = 0; - src.w = m_VideoWidth; - src.h = m_VideoHeight; + src.w = frame->width; + src.h = frame->height; dst.x = dst.y = 0; - dst.w = m_DisplayWidth; - dst.h = m_DisplayHeight; + dst.w = windowWidth; + dst.h = windowHeight; StreamUtils::scaleSourceToDestinationSurface(&src, &dst); @@ -733,6 +733,16 @@ VAAPIRenderer::renderFrame(AVFrame* frame) continue; } + SDL_Rect overlayRect = m_OverlayRect[type]; + + // Negative values are relative to the other side of the window + if (overlayRect.x < 0) { + overlayRect.x += windowWidth; + } + if (overlayRect.y < 0) { + overlayRect.y += windowHeight; + } + status = vaAssociateSubpicture(vaDeviceContext->display, m_OverlaySubpicture[type], &surface, @@ -741,10 +751,10 @@ VAAPIRenderer::renderFrame(AVFrame* frame) 0, m_OverlayImage[type].width, m_OverlayImage[type].height, - m_OverlayRect[type].x, - m_OverlayRect[type].y, - m_OverlayRect[type].w, - m_OverlayRect[type].h, + overlayRect.x, + overlayRect.y, + overlayRect.w, + overlayRect.h, 0); if (status == VA_STATUS_SUCCESS) { // Take temporary ownership of the overlay to prevent notifyOverlayUpdated() @@ -771,7 +781,7 @@ VAAPIRenderer::renderFrame(AVFrame* frame) surface, m_XWindow, 0, 0, - m_VideoWidth, m_VideoHeight, + frame->width, frame->height, dst.x, dst.y, dst.w, dst.h, NULL, 0, flags); diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.h b/app/streaming/video/ffmpeg-renderers/vaapi.h index 8f15cb41..834ec7b4 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.h +++ b/app/streaming/video/ffmpeg-renderers/vaapi.h @@ -111,11 +111,8 @@ private: int m_DrmFd; #endif - int m_VideoWidth; - int m_VideoHeight; + SDL_Window* m_Window; int m_VideoFormat; - int m_DisplayWidth; - int m_DisplayHeight; #ifdef HAVE_EGL enum class EglExportType {