Display a friendly error when trying to stream over 4K with H.264

app/streaming/session.cpp

@@ -628,6 +628,18 @@ bool Session::validateLaunch(SDL_Window* testWindow)
         emitLaunchWarning("An attached gamepad has no mapping and won't be usable. Visit the Moonlight help to resolve this.");
     }
 
+    // NVENC will fail to initialize when using dimensions over 4096 and H.264.
+    if (m_StreamConfig.width > 4096 || m_StreamConfig.height > 4096) {
+        if (m_Computer->maxLumaPixelsHEVC == 0) {
+            emit displayLaunchError("Your host PC's GPU doesn't support streaming video resolutions over 4K.");
+            return false;
+        }
+        else if (!m_StreamConfig.supportsHevc) {
+            emit displayLaunchError("Video resolutions over 4K are only supported by the HEVC codec.");
+            return false;
+        }
+    }
+
     if (m_Preferences->videoDecoderSelection == StreamingPreferences::VDS_FORCE_HARDWARE &&
             !m_StreamConfig.enableHdr && // HEVC Main10 was already checked for hardware decode support above
             !isHardwareDecodeAvailable(testWindow,