Replace the HEVC detection hack with a proper solution based on examining the RTSP DESCRIBE response

moonlight-common/src/com/limelight/nvstream/NvConnection.java

@@ -164,34 +164,9 @@ public class NvConnection {
 		// For now, always take the client's FPS request
 		context.negotiatedFps = context.streamConfig.getRefreshRate();
 		
-		// Determine whether we should request H.265 video
-		String gpuType = h.getGpuType(serverInfo);
-		if (gpuType == null || h.getMaxLumaPixelsHEVC(serverInfo) <= 0)  // Check if GFE version supports it
-		{
-			context.negotiatedVideoFormat = VideoFormat.H264;
-		}
-		else
-		{
-			// Check if GPU can do it (only 900-series non-M)
 		//
-			// This check is pretty broken. It's not handling mobile GPUs, but it
+		// Video stream format will be decided during the RTSP handshake
-			// only has false negatives which is absolutely required to avoid breaking
-			// streaming on GPUs with an H.265 compatible client.
 		//
-			// TODO: I think the correct way to do this is by examining the SDP attributes which
-			// should contain parameter sets for HEVC if the GPU supports it.
-			
-			gpuType = gpuType.toUpperCase();
-			if (context.streamConfig.getHevcSupported() && // Client wants it
-					((gpuType.contains("GTX 9") || gpuType.contains("GTX TITAN X")) && !gpuType.contains("M")))
-			{
-				context.negotiatedVideoFormat = VideoFormat.H265;
-			}
-			else 
-			{
-				context.negotiatedVideoFormat = VideoFormat.H264;
-			}
-		}
 		
 		NvApp app = context.streamConfig.getApp();
 		

moonlight-common/src/com/limelight/nvstream/rtsp/RtspConnection.java

@@ -7,6 +7,7 @@ import java.net.Socket;
 import java.util.HashMap;
 
 import com.limelight.nvstream.ConnectionContext;
+import com.limelight.nvstream.av.video.VideoDecoderRenderer.VideoFormat;
 import com.tinyrtsp.rtsp.message.RtspMessage;
 import com.tinyrtsp.rtsp.message.RtspRequest;
 import com.tinyrtsp.rtsp.message.RtspResponse;
@@ -106,6 +107,31 @@ public class RtspConnection {
 		return transactRtspMessage(m);
 	}
 	
+	private void processDescribeResponse(RtspResponse r) {
+		// The RTSP DESCRIBE reply will contain a collection of SDP media attributes that
+		// describe the various supported video stream formats and include the SPS, PPS,
+		// and VPS (if applicable). We will use this information to determine whether the
+		// server can support HEVC. For some reason, they still set the MIME type of the HEVC
+		// format to H264, so we can't just look for the HEVC MIME type. What we'll do instead is
+		// look for the base 64 encoded VPS NALU prefix that is unique to the HEVC bitstream.
+		String describeSdpContent = r.getPayload();
+		if (context.streamConfig.getHevcSupported() &&
+				describeSdpContent.contains("sprop-parameter-sets=AAAAAU")) {
+			context.negotiatedVideoFormat = VideoFormat.H265;
+		}
+		else {
+			context.negotiatedVideoFormat = VideoFormat.H264;
+		}
+	}
+	
+	private void processRtspSetupAudio(RtspResponse r) throws IOException {
+		try {
+			sessionId = Integer.parseInt(r.getOption("Session"));
+		} catch (NumberFormatException e) {
+			throw new IOException("RTSP SETUP response was malformed");
+		}
+	}
+	
 	public void doRtspHandshake() throws IOException {
 		RtspResponse r;
 		
@@ -119,16 +145,16 @@ public class RtspConnection {
 			throw new IOException("RTSP DESCRIBE request failed: "+r.getStatusCode());
 		}
 		
+		// Process the RTSP DESCRIBE response
+		processDescribeResponse(r);
+		
 		r = setupStream("audio");
 		if (r.getStatusCode() != 200) {
 			throw new IOException("RTSP SETUP request failed: "+r.getStatusCode());
 		}
 		
-		try {
+		// Process the RTSP SETUP streamid=audio response
-			sessionId = Integer.parseInt(r.getOption("Session"));
+		processRtspSetupAudio(r);
-		} catch (NumberFormatException e) {
-			throw new IOException("RTSP SETUP response was malformed");
-		}
 		
 		r = setupStream("video");
 		if (r.getStatusCode() != 200) {

moonlight-common/src/com/limelight/nvstream/rtsp/SdpGenerator.java

@@ -87,6 +87,16 @@ public class SdpGenerator {
 	}
 	
 	public static String generateSdpFromContext(ConnectionContext context) {
+		// By now, we must have decided on a format
+		if (context.negotiatedVideoFormat == VideoFormat.Unknown) {
+			throw new IllegalStateException("Video format negotiation must be completed before generating SDP response");
+		}
+		
+		// Also, resolution and frame rate must be set
+		if (context.negotiatedWidth == 0 || context.negotiatedHeight == 0 || context.negotiatedFps == 0) {
+			throw new IllegalStateException("Video resolution/FPS negotiation must be completed before generating SDP response");
+		}
+		
 		StringBuilder config = new StringBuilder();
 		config.append("v=0").append("\r\n"); // SDP Version 0
 		config.append("o=android 0 "+RtspConnection.getRtspVersionFromContext(context)+" IN ");