Support for X11 with software decoder using GLES 2.0 and EGL to perform YUV to RGB color conversion

CMakeLists.txt

@@ -29,13 +29,20 @@ pkg_check_modules(XLIB x11-xcb)
 pkg_check_modules(LIBVA vdpau)
 pkg_check_modules(PULSE libpulse-simple)
 pkg_check_modules(CEC libcec>=3.0.0)
+pkg_check_modules(EGL egl)
+pkg_check_modules(GLES glesv2)
 
-if(AVCODEC_FOUND AND AVUTIL_FOUND AND SDL_FOUND)
-  set(SOFTWARE_FOUND TRUE)
-  if(XLIB_FOUND AND LIBVA_FOUND)
-    set(VDPAU_FOUND TRUE)
-  else()
-    set(VDPAU_FOUND FALSE)
+if(AVCODEC_FOUND AND AVUTIL_FOUND)
+  if(EGL_FOUND AND GLES_FOUND AND XLIB_FOUND)
+    set(X11_FOUND TRUE)
+  endif()
+  if(SDL_FOUND OR X11_FOUND)
+    set(SOFTWARE_FOUND TRUE)
+    if(XLIB_FOUND AND LIBVA_FOUND)
+      set(VDPAU_FOUND TRUE)
+    else()
+      set(VDPAU_FOUND FALSE)
+    endif()
   endif()
 else()
   set(SOFTWARE_FOUND FALSE)
@@ -52,9 +59,17 @@ elseif(NOT AMLOGIC_FOUND AND NOT BROADCOM_FOUND AND NOT FREESCALE_FOUND AND NOT
 endif()
 
 if (SOFTWARE_FOUND)
-  list(APPEND SRC_LIST ./src/video/ffmpeg.c ./src/video/sdl.c ./src/audio/sdl.c)
-  list(APPEND MOONLIGHT_DEFINITIONS HAVE_SDL)
-  list(APPEND MOONLIGHT_OPTIONS SDL)
+  list(APPEND SRC_LIST ./src/video/ffmpeg.c)
+  if (SDL_FOUND)
+    list(APPEND SRC_LIST ./src/video/sdl.c ./src/audio/sdl.c)
+    list(APPEND MOONLIGHT_DEFINITIONS HAVE_SDL)
+    list(APPEND MOONLIGHT_OPTIONS SDL)
+  endif()
+  if (X11_FOUND)
+    list(APPEND SRC_LIST ./src/video/x11.c ./src/video/egl.c)
+    list(APPEND MOONLIGHT_DEFINITIONS HAVE_X11)
+    list(APPEND MOONLIGHT_OPTIONS X11)
+  endif()
   if(VDPAU_FOUND)
     list(APPEND SRC_LIST ./src/video/ffmpeg_vdpau.c)
     list(APPEND MOONLIGHT_DEFINITIONS HAVE_VDPAU)
@@ -120,9 +135,19 @@ if(FREESCALE_FOUND)
   install(TARGETS moonlight-imx DESTINATION ${CMAKE_INSTALL_LIBDIR})
 endif()
 
+if(SDL_FOUND)
+  target_include_directories(moonlight PRIVATE ${SDL_INCLUDE_DIRS})
+  target_link_libraries(moonlight ${SDL_LIBRARIES})
+endif()
+
+if(X11_FOUND)
+  target_include_directories(moonlight PRIVATE ${XLIB_INCLUDE_DIRS} ${EGL_INCLUDE_DIRS} ${GLES_INCLUDE_DIRS})
+  target_link_libraries(moonlight ${XLIB_LIBRARIES} ${EGL_LIBRARIES} ${GLES_LIBRARIES})
+endif()
+
 if (SOFTWARE_FOUND)
-  target_include_directories(moonlight PRIVATE ${SDL_INCLUDE_DIRS} ${AVCODEC_INCLUDE_DIRS} ${AVUTIL_INCLUDE_DIRS})
-  target_link_libraries(moonlight ${SDL_LIBRARIES} ${AVCODEC_LIBRARIES} ${AVUTIL_LIBRARIES})
+  target_include_directories(moonlight PRIVATE ${AVCODEC_INCLUDE_DIRS} ${AVUTIL_INCLUDE_DIRS})
+  target_link_libraries(moonlight ${AVCODEC_LIBRARIES} ${AVUTIL_LIBRARIES})
   if(VDPAU_FOUND)
     target_include_directories(moonlight PRIVATE ${XLIB_INCLUDE_DIRS} ${LIBVA_INCLUDE_DIRS})
     target_link_libraries(moonlight ${XLIB_LIBRARIES} ${LIBVA_LIBRARIES})

src/platform.c

@@ -55,6 +55,10 @@ enum platform platform_check(char* name) {
       return AML;
   }
   #endif
+  #ifdef HAVE_X11
+  if (std || strcmp(name, "x11") == 0)
+    return X11;
+  #endif
   #ifdef HAVE_SDL
   if (std || strcmp(name, "sdl") == 0)
     return SDL;
@@ -67,6 +71,10 @@ enum platform platform_check(char* name) {
 
 DECODER_RENDERER_CALLBACKS* platform_get_video(enum platform system) {
   switch (system) {
+  #ifdef HAVE_X11
+  case X11:
+    return &decoder_callbacks_x11;
+  #endif
   #ifdef HAVE_SDL
   case SDL:
     return &decoder_callbacks_sdl;

src/platform.h

@@ -26,7 +26,7 @@
 
 #define IS_EMBEDDED(SYSTEM) SYSTEM != SDL
 
-enum platform { NONE, SDL, PI, IMX, AML, FAKE };
+enum platform { NONE, SDL, X11, PI, IMX, AML, FAKE };
 
 enum platform platform_check(char*);
 PDECODER_RENDERER_CALLBACKS platform_get_video(enum platform system);
@@ -35,6 +35,9 @@ bool platform_supports_hevc(enum platform system);
 
 extern DECODER_RENDERER_CALLBACKS decoder_callbacks_fake;
 extern AUDIO_RENDERER_CALLBACKS audio_callbacks_fake;
+#ifdef HAVE_X11
+extern DECODER_RENDERER_CALLBACKS decoder_callbacks_x11;
+#endif
 #ifdef HAVE_SDL
 extern DECODER_RENDERER_CALLBACKS decoder_callbacks_sdl;
 void sdl_loop();

src/video/egl.c

@@ -0,0 +1,199 @@
+/*
+ * This file is part of Moonlight Embedded.
+ *
+ * Copyright (C) 2017 Iwan Timmer
+ *
+ * Moonlight is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Moonlight is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Moonlight; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "egl.h"
+
+#include <Limelight.h>
+
+#include <GLES2/gl2.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <pthread.h>
+#include <unistd.h>
+
+static const EGLint context_attributes[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
+static const char* texture_mappings[] = { "ymap", "umap", "vmap" };
+static const char* vertex_source = "\
+attribute vec2 position;\
+varying mediump vec2 tex_position;\
+\
+void main() {\
+  gl_Position = vec4(position, 0, 1);\
+  tex_position = vec2((position.x + 1.) / 2., (1. - position.y) / 2.);\
+}\
+";
+
+static const char* fragment_source = "\
+uniform lowp sampler2D ymap;\
+uniform lowp sampler2D umap;\
+uniform lowp sampler2D vmap;\
+varying mediump vec2 tex_position;\
+\
+void main() {\
+  mediump float y = texture2D(ymap, tex_position).r;\
+  mediump float u = texture2D(umap, tex_position).r - .5;\n\
+  mediump float v = texture2D(vmap, tex_position).r - .5;\n\
+  lowp float r = y + 1.28033 * v;\
+  lowp float g = y - .21482 * u - .38059 * v;\
+  lowp float b = y + 2.12798 * u;\
+  gl_FragColor = vec4(r, g, b, 1.0);\
+}\
+";
+
+static const float vertices[] = {
+  -1.f,  1.f,
+  -1.f, -1.f,
+  1.f, -1.f,
+  1.f, 1.f
+};
+
+static const GLuint elements[] = {
+  0, 1, 2,
+  2, 3, 0
+};
+
+static EGLDisplay display;
+static EGLSurface surface;
+static EGLContext context;
+
+static int width, height;
+static bool current;
+
+static GLuint texture_id[3], texture_uniform[3];
+static GLuint shader_program;
+
+void egl_init(EGLNativeDisplayType native_display, NativeWindowType native_window, int display_width, int display_height) {
+  width = display_width;
+  height = display_height;
+
+  // get an EGL display connection
+  display = eglGetDisplay(native_display);
+  if (display == EGL_NO_DISPLAY) {
+    fprintf( stderr, "EGL: error get display\n" );
+    exit(EXIT_FAILURE);
+  }
+
+  // initialize the EGL display connection
+  int major, minor;
+  EGLBoolean result = eglInitialize(display, &major, &minor);
+  if (result == EGL_FALSE) {
+    fprintf( stderr, "EGL: error initialising display\n");
+    exit(EXIT_FAILURE);
+  }
+
+  // get our config from the config class
+  EGLConfig config = NULL;
+  static const EGLint attribute_list[] = { EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_NONE };
+
+  EGLint totalConfigsFound = 0;
+  result = eglChooseConfig(display, attribute_list, &config, 1, &totalConfigsFound);
+  if (result != EGL_TRUE || totalConfigsFound == 0) {
+    fprintf(stderr, "EGL: Unable to query for available configs, found %d.\n", totalConfigsFound);
+    exit(EXIT_FAILURE);
+  }
+
+  // bind the OpenGL API to the EGL
+  result = eglBindAPI(EGL_OPENGL_ES_API);
+  if (result == EGL_FALSE) {
+    fprintf(stderr, "EGL: error binding API\n");
+    exit(EXIT_FAILURE);
+  }
+
+  // create an EGL rendering context
+  context = eglCreateContext(display, config, EGL_NO_CONTEXT, context_attributes);
+  if (context == EGL_NO_CONTEXT) {
+    fprintf(stderr, "EGL: couldn't get a valid context\n");
+    exit(EXIT_FAILURE);
+  }
+
+  // finally we can create a new surface using this config and window
+  surface = eglCreateWindowSurface(display, config, (NativeWindowType) native_window, NULL);
+  eglMakeCurrent(display, surface, surface, context);
+
+  glEnable(GL_TEXTURE_2D);
+
+  GLuint vbo;
+  glGenBuffers(1, &vbo);
+  glBindBuffer(GL_ARRAY_BUFFER, vbo);
+  glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+
+  GLuint ebo;
+  glGenBuffers(1, &ebo);
+  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+  glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(elements), elements, GL_STATIC_DRAW);
+  
+  GLuint vertex_shader = glCreateShader(GL_VERTEX_SHADER);
+  glShaderSource(vertex_shader, 1, &vertex_source, NULL);
+  glCompileShader(vertex_shader);
+  GLint maxLength = 0;
+
+  GLuint fragment_shader = glCreateShader(GL_FRAGMENT_SHADER);
+  glShaderSource(fragment_shader, 1, &fragment_source, NULL);
+  glCompileShader(fragment_shader);
+
+  shader_program = glCreateProgram();
+  glAttachShader(shader_program, vertex_shader);
+  glAttachShader(shader_program, fragment_shader);
+
+  glLinkProgram(shader_program);
+  glBindAttribLocation(shader_program, 0, "position");
+  glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
+
+  glGenTextures(3, texture_id);
+  for (int i = 0; i < 3; i++) {
+    glBindTexture(GL_TEXTURE_2D, texture_id[i]);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, i > 0 ? width / 2 : width, i > 0 ? height / 2 : height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0);
+
+    texture_uniform[i] = glGetUniformLocation(shader_program, texture_mappings[i]);
+  }
+
+  eglMakeCurrent(display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+}
+
+void egl_draw(const uint8_t* image[3]) {
+  if (!current) {
+    eglMakeCurrent(display, surface, surface, context);
+    current = True;
+  }
+
+  glUseProgram(shader_program);
+  glEnableVertexAttribArray(0);
+
+  for (int i = 0; i < 3; i++) {
+    glActiveTexture(GL_TEXTURE0 + i);
+    glBindTexture(GL_TEXTURE_2D, texture_id[i]);
+    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, i > 0 ? width / 2 : width, i > 0 ? height / 2 : height, GL_LUMINANCE, GL_UNSIGNED_BYTE, image[i]);
+    glUniform1i(texture_uniform[i], i);
+  }
+
+  glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
+
+  eglSwapBuffers(display, surface);
+}
+
+void egl_destroy() {
+  eglMakeCurrent(display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+  eglDestroySurface(display, surface);
+  eglDestroyContext(display, context);
+  eglTerminate(display);
+}

src/video/egl.h

@@ -0,0 +1,24 @@
+/*
+ * This file is part of Moonlight Embedded.
+ *
+ * Copyright (C) 2017 Iwan Timmer
+ *
+ * Moonlight is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Moonlight is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Moonlight; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <EGL/egl.h>
+
+void egl_init(EGLNativeDisplayType native_display, NativeWindowType native_window, int display_width, int display_height);
+void egl_draw(const uint8_t* image[3]);
+void egl_destroy();

src/video/x11.c

@@ -0,0 +1,97 @@
+/*
+ * This file is part of Moonlight Embedded.
+ *
+ * Copyright (C) 2017 Iwan Timmer
+ *
+ * Moonlight is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Moonlight is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Moonlight; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "../video.h"
+#include "egl.h"
+#include "ffmpeg.h"
+
+#include <Limelight.h>
+
+#include <X11/Xlib.h>
+#include <X11/Xatom.h>
+#include <X11/Xutil.h>
+
+#include <stdbool.h>
+
+#define DECODER_BUFFER_SIZE 92*1024
+
+static char* ffmpeg_buffer = NULL;
+
+static Display *display;
+
+void x11_setup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) {
+  int avc_flags = SLICE_THREADING;
+  if (drFlags & FORCE_HARDWARE_ACCELERATION)
+    avc_flags |= HARDWARE_ACCELERATION;
+
+  if (ffmpeg_init(videoFormat, width, height, avc_flags, 2, 2) < 0) {
+    fprintf(stderr, "Couldn't initialize video decoding\n");
+    exit(1);
+  }
+
+  ffmpeg_buffer = malloc(DECODER_BUFFER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE);
+  if (ffmpeg_buffer == NULL) {
+    fprintf(stderr, "Not enough memory\n");
+    exit(1);
+  }
+
+  display = XOpenDisplay(NULL);
+  if (!display) {
+    fprintf(stderr, "Error: failed to open X display.\n");
+    return;
+  }
+
+  Window root = DefaultRootWindow(display);
+  XSetWindowAttributes winattr = {0};
+  Window window = XCreateWindow(display, root, 0, 0, width, height, 0, CopyFromParent, InputOutput, CopyFromParent, CWEventMask, &winattr);
+  XMapWindow(display, window);
+  XStoreName(display, window, "Moonlight");
+
+  egl_init(display, window, width, height);
+}
+
+void x11_cleanup() {
+  ffmpeg_destroy();
+  egl_destroy();
+}
+
+int x11_submit_decode_unit(PDECODE_UNIT decodeUnit) {
+  if (decodeUnit->fullLength < DECODER_BUFFER_SIZE) {
+    PLENTRY entry = decodeUnit->bufferList;
+    int length = 0;
+    while (entry != NULL) {
+      memcpy(ffmpeg_buffer+length, entry->data, entry->length);
+      length += entry->length;
+      entry = entry->next;
+    }
+    ffmpeg_decode(ffmpeg_buffer, length);
+    AVFrame* frame = ffmpeg_get_frame();
+    if (frame != NULL)
+      egl_draw(frame->data);
+  }
+
+  return DR_OK;
+}
+
+DECODER_RENDERER_CALLBACKS decoder_callbacks_x11 = {
+  .setup = x11_setup,
+  .cleanup = x11_cleanup,
+  .submitDecodeUnit = x11_submit_decode_unit,
+  .capabilities = CAPABILITY_SLICES_PER_FRAME(4) | CAPABILITY_REFERENCE_FRAME_INVALIDATION | CAPABILITY_DIRECT_SUBMIT,
+};