Don't call dlsym() in our DRM master hooks

Not only is it faster to cache the function pointers, calling
dlsym() inside open()/close() can lead to deadlocks when using
Vulkan Video decoding on top of the Nvidia driver.
This commit is contained in:
Cameron Gutman
2025-10-12 22:49:29 -05:00
parent ff81f74391
commit 1144dbccb3
3 changed files with 73 additions and 27 deletions

View File

@@ -7,7 +7,6 @@
#endif
#include "SDL_compat.h"
#include <dlfcn.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
@@ -100,7 +99,7 @@ int takeMasterFromSdlFd()
}
}
int openHook(const char *funcname, const char *pathname, int flags, va_list va)
int openHook(typeof(open) *real_open, typeof(close) *real_close, const char *pathname, int flags, va_list va)
{
int fd;
mode_t mode;
@@ -108,11 +107,11 @@ int openHook(const char *funcname, const char *pathname, int flags, va_list va)
// Call the real thing to do the open operation
if (__OPEN_NEEDS_MODE(flags)) {
mode = va_arg(va, mode_t);
fd = ((typeof(open)*)dlsym(RTLD_NEXT, funcname))(pathname, flags, mode);
fd = real_open(pathname, flags, mode);
}
else {
mode = 0;
fd = ((typeof(open)*)dlsym(RTLD_NEXT, funcname))(pathname, flags);
fd = real_open(pathname, flags);
}
// If the file was successfully opened and we have a DRM master FD,
@@ -146,7 +145,7 @@ int openHook(const char *funcname, const char *pathname, int flags, va_list va)
allocatedFdIndex = getSdlFdEntryIndex(false);
if (allocatedFdIndex >= 0) {
// Close fd that we opened earlier (skipping our close() hook)
((typeof(close)*)dlsym(RTLD_NEXT, "close"))(fd);
real_close(fd);
// dup() an existing FD into the unused slot
fd = dup(g_SdlDrmMasterFds[allocatedFdIndex]);
@@ -163,16 +162,16 @@ int openHook(const char *funcname, const char *pathname, int flags, va_list va)
}
// Close fd that we opened earlier (skipping our close() hook)
((typeof(close)*)dlsym(RTLD_NEXT, "close"))(fd);
real_close(fd);
// We are not allowed to call drmSetMaster() without CAP_SYS_ADMIN,
// but since we just dropped the master, we can become master by
// simply creating a new FD. Let's do it.
if (__OPEN_NEEDS_MODE(flags)) {
fd = ((typeof(open)*)dlsym(RTLD_NEXT, funcname))(pathname, flags, mode);
fd = real_open(pathname, flags, mode);
}
else {
fd = ((typeof(open)*)dlsym(RTLD_NEXT, funcname))(pathname, flags);
fd = real_open(pathname, flags);
}
}