Autodetect VDPAU support

src/platform.c

@@ -60,12 +60,14 @@ enum platform platform_check(char* name) {
   }
   #endif
   #ifdef HAVE_X11
-  if (std || strcmp(name, "x11") == 0)
+  if (std || strcmp(name, "x11") == 0 || strcmp(name, "x11_vdpau") == 0) {
+    int x11 = x11_init(strcmp(name, "x11") != 0);
+    #ifdef HAVE_VDPAU
+    if (strcmp(name, "x11") != 0 && x11 == 0)
+      return X11_VDPAU;
+    #endif
     return X11;
-  #ifdef HAVE_VDPAU
-  if (std || strcmp(name, "x11_vdpau") == 0)
-    return X11_VDPAU;
-  #endif
+  }
   #endif
   #ifdef HAVE_SDL
   if (std || strcmp(name, "sdl") == 0)

src/video/ffmpeg.c

@@ -46,7 +46,7 @@ enum decoders ffmpeg_decoder;
 
 // This function must be called before
 // any other decoding functions
-int ffmpeg_init(int videoFormat, int width, int height, int perf_lvl, int buffer_count, int thread_count, void* context) {
+int ffmpeg_init(int videoFormat, int width, int height, int perf_lvl, int buffer_count, int thread_count) {
   // Initialize the avcodec library and register codecs
   av_log_set_level(AV_LOG_QUIET);
   avcodec_register_all();
@@ -133,7 +133,7 @@ int ffmpeg_init(int videoFormat, int width, int height, int perf_lvl, int buffer
 
   #ifdef HAVE_VDPAU
   if (ffmpeg_decoder == VDPAU)
-    vdpau_init(decoder_ctx, (Display*) context, width, height);
+    vdpau_init(decoder_ctx, width, height);
   #endif
 
   return 0;

src/video/ffmpeg.h

@@ -39,7 +39,7 @@
 enum decoders {SOFTWARE, VDPAU};
 extern enum decoders ffmpeg_decoder;
 
-int ffmpeg_init(int videoFormat, int width, int height, int perf_lvl, int buffer_count, int thread_count, void* context);
+int ffmpeg_init(int videoFormat, int width, int height, int perf_lvl, int buffer_count, int thread_count);
 void ffmpeg_destroy(void);
 
 int ffmpeg_draw_frame(AVFrame *pict);

src/video/ffmpeg_vdpau.c

@@ -41,6 +41,7 @@ static struct vdpau_render_state* vdp_render_state[MAX_RENDER_STATES];
 static int vdp_render_states = 0;
 
 static VdpGetProcAddress* vdp_get_proc_address;
+static VdpDeviceDestroy* vdp_device_destroy;
 static VdpDecoderCreate* vdp_decoder_create;
 static VdpDecoderRender* vdp_decoder_render;
 static VdpVideoSurfaceGetBitsYCbCr* vdp_video_surface_get_bits_y_cb_cr;
@@ -96,14 +97,16 @@ static void vdp_draw_horiz_band(struct AVCodecContext* context, const AVFrame* f
   vdp_decoder_render(vdp_decoder, render_state->surface, (VdpPictureInfo const*)&(render_state->info), render_state->bitstream_buffers_used, render_state->bitstream_buffers);
 }
 
-int vdpau_init(AVCodecContext* decoder_ctx, Display* display, int width, int height) {
-  if (vdp_device)
-    return vdp_device;
-
+int vdpau_init_lib(Display* display) {
   VdpStatus status = vdp_device_create_x11(display, DefaultScreen(display), &vdp_device, &vdp_get_proc_address);
   if (status != VDP_STATUS_OK)
      return -1;
 
+  vdp_get_proc_address(vdp_device, VDP_FUNC_ID_DEVICE_DESTROY, (void**)&vdp_device_destroy);
+  return 0;  
+}
+
+int vdpau_init(AVCodecContext* decoder_ctx, int width, int height) {
   vdp_get_proc_address(vdp_device, VDP_FUNC_ID_VIDEO_SURFACE_GET_BITS_Y_CB_CR, (void**)&vdp_video_surface_get_bits_y_cb_cr);
   vdp_get_proc_address(vdp_device, VDP_FUNC_ID_VIDEO_SURFACE_CREATE, (void**)&vdp_video_surface_create);
   vdp_get_proc_address(vdp_device, VDP_FUNC_ID_OUTPUT_SURFACE_CREATE, (void**)&vdp_output_surface_create);
@@ -134,7 +137,7 @@ int vdpau_init(AVCodecContext* decoder_ctx, Display* display, int width, int hei
     return -1;
   }
 
-  status = vdp_decoder_create(vdp_device, VDP_DECODER_PROFILE_H264_HIGH, width, height, 16, &vdp_decoder);
+  VdpStatus status = vdp_decoder_create(vdp_device, VDP_DECODER_PROFILE_H264_HIGH, width, height, 16, &vdp_decoder);
   if (status != VDP_STATUS_OK) {
     printf("Can't create VDPAU decoder\n");
     return -1;
@@ -143,6 +146,10 @@ int vdpau_init(AVCodecContext* decoder_ctx, Display* display, int width, int hei
   return vdp_device;
 }
 
+void vdpau_destroy() {
+  vdp_device_destroy(vdp_device);
+}
+
 AVFrame* vdpau_get_frame(AVFrame* dec_frame) {
   struct vdpau_render_state *render_state = (struct vdpau_render_state *)dec_frame->data[0];
   void *dest[3] = {

src/video/ffmpeg_vdpau.h

@@ -20,7 +20,9 @@
 #include <X11/Xlib.h>
 #include <libavcodec/avcodec.h>
 
-int vdpau_init(AVCodecContext* decoder_ctx, Display* display, int width, int height);
+int vdpau_init_lib(Display* display);
+int vdpau_init(AVCodecContext* decoder_ctx, int width, int height);
+void vdpau_destroy();
 AVFrame* vdpau_get_frame(AVFrame* dec_frame);
 int vdpau_init_presentation(Drawable win, int width, int height);
 void vdpau_queue(AVFrame* dec_frame);

src/video/sdl.c

@@ -35,7 +35,7 @@ static char* ffmpeg_buffer;
 static int sdl_setup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) {
   int avc_flags = SLICE_THREADING;
 
-  if (ffmpeg_init(videoFormat, width, height, avc_flags, SDL_BUFFER_FRAMES, sysconf(_SC_NPROCESSORS_ONLN), NULL) < 0) {
+  if (ffmpeg_init(videoFormat, width, height, avc_flags, SDL_BUFFER_FRAMES, sysconf(_SC_NPROCESSORS_ONLN)) < 0) {
     fprintf(stderr, "Couldn't initialize video decoding\n");
     return -1;
   }

src/video/video.h

@@ -19,10 +19,13 @@
 
 #include <Limelight.h>
 
+#include <stdbool.h>
+
 #define DISPLAY_FULLSCREEN 1
 #define ENABLE_HARDWARE_ACCELERATION 2
 
 #ifdef HAVE_X11
+int x11_init(bool vdpau);
 extern DECODER_RENDERER_CALLBACKS decoder_callbacks_x11;
 #ifdef HAVE_VDPAU
 extern DECODER_RENDERER_CALLBACKS decoder_callbacks_x11_vdpau;

src/video/x11.c

@@ -20,7 +20,9 @@
 #include "video.h"
 #include "egl.h"
 #include "ffmpeg.h"
+#ifdef HAVE_VDPAU
 #include "ffmpeg_vdpau.h"
+#endif
 
 #include "../input/x11.h"
 #include "../loop.h"
@@ -37,7 +39,7 @@
 
 static char* ffmpeg_buffer = NULL;
 
-static Display *display;
+static Display *display = NULL;
 
 static int pipefd[2];
 
@@ -50,23 +52,30 @@ static int frame_handle(int pipefd) {
   return LOOP_OK;
 }
 
-int x11_setup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) {
-  int avc_flags = SLICE_THREADING;
-  if (drFlags & ENABLE_HARDWARE_ACCELERATION)
-    avc_flags |= HARDWARE_ACCELERATION;
+int x11_init(bool vdpau) {
+  XInitThreads();
+  display = XOpenDisplay(NULL);
+  if (!display)
+    return -1;
 
+  #ifdef HAVE_VDPAU
+  if (vdpau && vdpau_init_lib(display) != 0)
+    return -2;
+  #endif
+
+  return 0;
+}
+
+int x11_setup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) {
   ffmpeg_buffer = malloc(DECODER_BUFFER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE);
   if (ffmpeg_buffer == NULL) {
     fprintf(stderr, "Not enough memory\n");
-    ffmpeg_destroy();
     return -1;
   }
 
-  XInitThreads();
-  display = XOpenDisplay(NULL);
   if (!display) {
     fprintf(stderr, "Error: failed to open X display.\n");
-    return -2;
+    return -1;
   }
 
   Window root = DefaultRootWindow(display);
@@ -90,16 +99,25 @@ int x11_setup(int videoFormat, int width, int height, int redrawRate, void* cont
 
     XSendEvent(display, DefaultRootWindow(display), False, SubstructureRedirectMask | SubstructureNotifyMask, &xev);
   }
+  XFlush(display);
 
-  if (ffmpeg_init(videoFormat, width, height, avc_flags, 2, 2, display) < 0) {
+  int avc_flags = SLICE_THREADING;
+  #ifdef HAVE_VDPAU
+  if (drFlags & ENABLE_HARDWARE_ACCELERATION)
+    avc_flags |= HARDWARE_ACCELERATION;
+  #endif
+
+  if (ffmpeg_init(videoFormat, width, height, avc_flags, 2, 2) < 0) {
     fprintf(stderr, "Couldn't initialize video decoding\n");
     return -1;
   }
 
   if (ffmpeg_decoder == SOFTWARE)
     egl_init(display, window, width, height);
-  else
+  #ifdef HAVE_VDPAU
+  else if (ffmpeg_decoder == VDPAU)
     vdpau_init_presentation(window, width, height);
+  #endif
 
   x11_input_init(display, window);
 
@@ -114,7 +132,7 @@ int x11_setup(int videoFormat, int width, int height, int redrawRate, void* cont
 }
 
 int x11_setup_vdpau(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) {
-  x11_setup(videoFormat, width, height, redrawRate, context, drFlags | ENABLE_HARDWARE_ACCELERATION);
+  return x11_setup(videoFormat, width, height, redrawRate, context, drFlags | ENABLE_HARDWARE_ACCELERATION);
 }
 
 void x11_cleanup() {