Add support for Generation 5 servers (GFE 2.10.2+)

This commit is contained in:
Cameron Gutman 2016-02-19 03:41:03 -05:00
parent 5718c47be7
commit d9cb5eacf8
13 changed files with 216 additions and 90 deletions

View File

@ -14,6 +14,9 @@ public class ConnectionContext {
// Gen 4 servers are 2.2.2+
public static final int SERVER_GENERATION_4 = 4;
// Gen 5 servers are 2.10.2+
public static final int SERVER_GENERATION_5 = 5;
public InetAddress serverAddress;
public StreamConfiguration streamConfig;
public VideoDecoderRenderer videoDecoderRenderer;

View File

@ -116,7 +116,7 @@ public class NvConnection {
context.connListener.displayMessage("This app requires GeForce Experience 2.2.2 or later. Please upgrade GFE on your PC and try again.");
return false;
}
else if (majorVersion > 4) {
else if (majorVersion > 5) {
// Warn the user but allow them to continue
context.connListener.displayTransientMessage("This version of GFE is not currently supported. You may experience issues until this app is updated.");
}
@ -126,9 +126,12 @@ public class NvConnection {
context.serverGeneration = ConnectionContext.SERVER_GENERATION_3;
break;
case 4:
default:
context.serverGeneration = ConnectionContext.SERVER_GENERATION_4;
break;
case 5:
default:
context.serverGeneration = ConnectionContext.SERVER_GENERATION_5;
break;
}
LimeLog.info("Server major version: "+majorVersion);

View File

@ -7,7 +7,9 @@ public interface ConnectionStatusListener {
public void connectionSinkTooSlow(int firstLostFrame, int nextSuccessfulFrame);
public void connectionReceivedFrame(int frameIndex);
public void connectionReceivedCompleteFrame(int frameIndex);
public void connectionSawFrame(int frameIndex);
public void connectionLostPackets(int lastReceivedPacket, int nextReceivedPacket);
}

View File

@ -43,11 +43,22 @@ public class VideoDepacketizer {
private static final int DU_LIMIT = 15;
private AbstractPopulatedBufferList<DecodeUnit> decodedUnits;
private final int frameHeaderOffset;
public VideoDepacketizer(ConnectionContext context, ConnectionStatusListener controlListener, int nominalPacketSize)
{
this.controlListener = controlListener;
this.nominalPacketDataLength = nominalPacketSize - VideoPacket.HEADER_SIZE;
if (context.serverGeneration >= ConnectionContext.SERVER_GENERATION_5) {
// Gen 5 servers have an 8 byte header in the data portion of the first
// packet of each frame
frameHeaderOffset = 8;
}
else {
frameHeaderOffset = 0;
}
boolean unsynchronized;
if (context.videoDecoderRenderer != null) {
int videoCaps = context.videoDecoderRenderer.getCapabilities();
@ -177,7 +188,7 @@ public class VideoDepacketizer {
// Packets now owned by the DU
backingPacketTail = backingPacketHead = null;
controlListener.connectionReceivedFrame(frameNumber);
controlListener.connectionReceivedCompleteFrame(frameNumber);
// Submit the DU to the consumer
decodedUnits.addPopulatedObject(du);
@ -367,6 +378,9 @@ public class VideoDepacketizer {
return;
}
// Notify the listener of the latest frame we've seen from the PC
controlListener.connectionSawFrame(frameIndex);
// Look for a frame start before receiving a frame end
if (firstPacket && decodingFrame)
{
@ -446,6 +460,12 @@ public class VideoDepacketizer {
}
lastPacketInStream = streamPacketIndex;
// If this is the first packet, skip the frame header (if one exists)
if (firstPacket) {
cachedReassemblyDesc.offset += frameHeaderOffset;
cachedReassemblyDesc.length -= frameHeaderOffset;
}
if (firstPacket && isIdrFrameStart(cachedReassemblyDesc))
{
// The slow path doesn't update the frame start time by itself

View File

@ -41,6 +41,13 @@ public class ControlStream implements ConnectionStatusListener {
0x060a, // Loss Stats
0x0611, // Frame Stats (unused)
};
private static final short packetTypesGen5[] = {
0x0305, // Start A
0x0307, // Start B
0x0301, // Invalidate reference frames
0x0201, // Loss Stats
0x0204, // Frame Stats (unused)
};
private static final short payloadLengthsGen3[] = {
-1, // Start A
@ -56,6 +63,13 @@ public class ControlStream implements ConnectionStatusListener {
32, // Loss Stats
64, // Frame Stats
};
private static final short payloadLengthsGen5[] = {
-1, // Start A
16, // Start B
24, // Invalidate reference frames
32, // Loss Stats
80, // Frame Stats
};
private static final byte[] precontructedPayloadsGen3[] = {
new byte[]{0}, // Start A
@ -71,10 +85,18 @@ public class ControlStream implements ConnectionStatusListener {
null, // Loss Stats
null, // Frame Stats
};
private static final byte[] precontructedPayloadsGen5[] = {
new byte[]{0, 0}, // Start A
null, // Start B
null, // Invalidate reference frames
null, // Loss Stats
null, // Frame Stats
};
public static final int LOSS_REPORT_INTERVAL_MS = 50;
private int currentFrame;
private int lastGoodFrame;
private int lastSeenFrame;
private int lossCountSinceLastReport;
private ConnectionContext context;
@ -121,11 +143,16 @@ public class ControlStream implements ConnectionStatusListener {
preconstructedPayloads = precontructedPayloadsGen3;
break;
case ConnectionContext.SERVER_GENERATION_4:
default:
packetTypes = packetTypesGen4;
payloadLengths = payloadLengthsGen4;
preconstructedPayloads = precontructedPayloadsGen4;
break;
case ConnectionContext.SERVER_GENERATION_5:
default:
packetTypes = packetTypesGen5;
payloadLengths = payloadLengthsGen5;
preconstructedPayloads = precontructedPayloadsGen5;
break;
}
if (context.videoDecoderRenderer != null) {
@ -164,7 +191,7 @@ public class ControlStream implements ConnectionStatusListener {
bb.putInt(lossCountSinceLastReport); // Packet loss count
bb.putInt(LOSS_REPORT_INTERVAL_MS); // Time since last report in milliseconds
bb.putInt(1000);
bb.putLong(currentFrame); // Last successfully received frame
bb.putLong(lastGoodFrame); // Last successfully received frame
bb.putInt(0);
bb.putInt(0);
bb.putInt(0x14);
@ -315,7 +342,8 @@ public class ControlStream implements ConnectionStatusListener {
private ControlStream.NvCtlResponse doStartB() throws IOException
{
if (context.serverGeneration == ConnectionContext.SERVER_GENERATION_3) {
// Gen 3 and 5 both use a packet of this form
if (context.serverGeneration != ConnectionContext.SERVER_GENERATION_4) {
ByteBuffer payload = ByteBuffer.wrap(new byte[payloadLengths[IDX_START_B]]).order(ByteOrder.LITTLE_ENDIAN);
payload.putInt(0);
@ -334,16 +362,28 @@ public class ControlStream implements ConnectionStatusListener {
}
private void requestIdrFrame() throws IOException {
// On Gen 3, we use the invalidate reference frames trick which works for about 5 hours of streaming at 60 FPS
// On Gen 3, we use the invalidate reference frames trick.
// On Gen 4+, we use the known IDR frame request packet
// On Gen 5, we're currently using the invalidate reference frames trick again.
if (context.serverGeneration == ConnectionContext.SERVER_GENERATION_3) {
if (context.serverGeneration != ConnectionContext.SERVER_GENERATION_4) {
ByteBuffer conf = ByteBuffer.wrap(new byte[payloadLengths[IDX_INVALIDATE_REF_FRAMES]]).order(ByteOrder.LITTLE_ENDIAN);
//conf.putLong(firstLostFrame);
//conf.putLong(nextSuccessfulFrame);
conf.putLong(0);
conf.putLong(0xFFFFF);
// Early on, we'll use a special IDR sequence. Otherwise,
// we'll just say we lost the last 32 frames. This is larger
// than the number of buffered frames in the encoder (16) so
// it should trigger an IDR frame.
if (lastSeenFrame < 0x20) {
conf.putLong(0);
conf.putLong(0x20);
}
else {
conf.putLong(lastSeenFrame - 0x20);
conf.putLong(lastSeenFrame);
}
conf.putLong(0);
sendAndGetReply(new NvCtlPacket(packetTypes[IDX_INVALIDATE_REF_FRAMES],
@ -534,7 +574,7 @@ public class ControlStream implements ConnectionStatusListener {
// Suppress connection warnings for the first 150 frames to allow the connection
// to stabilize
if (currentFrame < 150) {
if (lastGoodFrame < 150) {
return;
}
@ -569,7 +609,7 @@ public class ControlStream implements ConnectionStatusListener {
// Suppress connection warnings for the first 150 frames to allow the connection
// to stabilize
if (currentFrame < 150) {
if (lastGoodFrame < 150) {
return;
}
@ -579,8 +619,12 @@ public class ControlStream implements ConnectionStatusListener {
}
}
public void connectionReceivedFrame(int frameIndex) {
currentFrame = frameIndex;
public void connectionReceivedCompleteFrame(int frameIndex) {
lastGoodFrame = frameIndex;
}
public void connectionSawFrame(int frameIndex) {
lastSeenFrame = frameIndex;
}
public void connectionLostPackets(int lastReceivedPacket, int nextReceivedPacket) {

View File

@ -271,7 +271,7 @@ public class ControllerStream {
}
else {
// Use multi-controller packets for generation 4 and above
queuePacket(new MultiControllerPacket((short) 0, buttonFlags, leftTrigger,
queuePacket(new MultiControllerPacket(context, (short) 0, buttonFlags, leftTrigger,
rightTrigger, leftStickX, leftStickY,
rightStickX, rightStickY));
}
@ -288,7 +288,7 @@ public class ControllerStream {
}
else {
// Use multi-controller packets for generation 4 and above
queuePacket(new MultiControllerPacket(controllerNumber, buttonFlags, leftTrigger,
queuePacket(new MultiControllerPacket(context, controllerNumber, buttonFlags, leftTrigger,
rightTrigger, leftStickX, leftStickY,
rightStickX, rightStickY));
}
@ -296,17 +296,17 @@ public class ControllerStream {
public void sendMouseButtonDown(byte mouseButton)
{
queuePacket(new MouseButtonPacket(true, mouseButton));
queuePacket(new MouseButtonPacket(context, true, mouseButton));
}
public void sendMouseButtonUp(byte mouseButton)
{
queuePacket(new MouseButtonPacket(false, mouseButton));
queuePacket(new MouseButtonPacket(context, false, mouseButton));
}
public void sendMouseMove(short deltaX, short deltaY)
{
queuePacket(new MouseMovePacket(deltaX, deltaY));
queuePacket(new MouseMovePacket(context, deltaX, deltaY));
}
public void sendKeyboardInput(short keyMap, byte keyDirection, byte modifier)
@ -316,6 +316,6 @@ public class ControllerStream {
public void sendMouseScroll(byte scrollClicks)
{
queuePacket(new MouseScrollPacket(scrollClicks));
queuePacket(new MouseScrollPacket(context, scrollClicks));
}
}

View File

@ -14,9 +14,9 @@ public class KeyboardPacket extends InputPacket {
public static final byte MODIFIER_CTRL = 0x02;
public static final byte MODIFIER_ALT = 0x04;
short keyCode;
byte keyDirection;
byte modifier;
private short keyCode;
private byte keyDirection;
private byte modifier;
public KeyboardPacket(short keyCode, byte keyDirection, byte modifier) {
super(PACKET_TYPE);

View File

@ -3,6 +3,8 @@ package com.limelight.nvstream.input;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.limelight.nvstream.ConnectionContext;
public class MouseButtonPacket extends InputPacket {
byte buttonEventType;
@ -20,7 +22,7 @@ public class MouseButtonPacket extends InputPacket {
public static final byte BUTTON_MIDDLE = 0x02;
public static final byte BUTTON_RIGHT = 0x03;
public MouseButtonPacket(boolean buttonDown, byte mouseButton)
public MouseButtonPacket(ConnectionContext context, boolean buttonDown, byte mouseButton)
{
super(PACKET_TYPE);
@ -28,6 +30,11 @@ public class MouseButtonPacket extends InputPacket {
buttonEventType = buttonDown ?
PRESS_EVENT : RELEASE_EVENT;
// On Gen 5 servers, the button event codes are incremented by one
if (context.serverGeneration >= ConnectionContext.SERVER_GENERATION_5) {
buttonEventType++;
}
}
@Override

View File

@ -3,36 +3,40 @@ package com.limelight.nvstream.input;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
public class MouseMovePacket extends InputPacket {
private static final byte[] HEADER =
{
0x06,
0x00,
0x00,
0x00
};
import com.limelight.nvstream.ConnectionContext;
public static final int PACKET_TYPE = 0x8;
public static final int PAYLOAD_LENGTH = 8;
public static final int PACKET_LENGTH = PAYLOAD_LENGTH +
public class MouseMovePacket extends InputPacket {
private static final int HEADER_CODE = 0x06;
private static final int PACKET_TYPE = 0x8;
private static final int PAYLOAD_LENGTH = 8;
private static final int PACKET_LENGTH = PAYLOAD_LENGTH +
InputPacket.HEADER_LENGTH;
private int headerCode;
// Accessed in ControllerStream for batching
short deltaX;
short deltaY;
public MouseMovePacket(short deltaX, short deltaY)
public MouseMovePacket(ConnectionContext context, short deltaX, short deltaY)
{
super(PACKET_TYPE);
this.headerCode = HEADER_CODE;
// On Gen 5 servers, the header code is incremented by one
if (context.serverGeneration >= ConnectionContext.SERVER_GENERATION_5) {
headerCode++;
}
this.deltaX = deltaX;
this.deltaY = deltaY;
}
@Override
public void toWirePayload(ByteBuffer bb) {
bb.order(ByteOrder.LITTLE_ENDIAN).putInt(headerCode);
bb.order(ByteOrder.BIG_ENDIAN);
bb.put(HEADER);
bb.putShort(deltaX);
bb.putShort(deltaY);
}

View File

@ -3,28 +3,38 @@ package com.limelight.nvstream.input;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.limelight.nvstream.ConnectionContext;
public class MouseScrollPacket extends InputPacket {
private static final int HEADER_CODE = 0x09;
private static final int PACKET_TYPE = 0xa;
private static final int PAYLOAD_LENGTH = 10;
private static final int PACKET_LENGTH = PAYLOAD_LENGTH +
InputPacket.HEADER_LENGTH;
short scroll;
public MouseScrollPacket(byte scrollClicks)
private int headerCode;
private short scroll;
public MouseScrollPacket(ConnectionContext context, byte scrollClicks)
{
super(PACKET_TYPE);
this.headerCode = HEADER_CODE;
// On Gen 5 servers, the header code is incremented by one
if (context.serverGeneration >= ConnectionContext.SERVER_GENERATION_5) {
headerCode++;
}
this.scroll = (short)(scrollClicks * 120);
}
@Override
public void toWirePayload(ByteBuffer bb) {
bb.order(ByteOrder.BIG_ENDIAN);
public void toWirePayload(ByteBuffer bb) {
bb.order(ByteOrder.LITTLE_ENDIAN).putInt(headerCode);
bb.put((byte) 0x09);
bb.put((byte) 0);
bb.put((byte) 0);
bb.put((byte) 0);
bb.order(ByteOrder.BIG_ENDIAN);
bb.putShort(scroll);
bb.putShort(scroll);

View File

@ -3,6 +3,8 @@ package com.limelight.nvstream.input;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.limelight.nvstream.ConnectionContext;
public class MultiControllerPacket extends InputPacket {
private static final byte[] TAIL =
{
@ -14,6 +16,7 @@ public class MultiControllerPacket extends InputPacket {
0x00
};
private static final int HEADER_CODE = 0x0d;
private static final int PACKET_TYPE = 0x1e;
private static final short PAYLOAD_LENGTH = 30;
@ -29,12 +32,22 @@ public class MultiControllerPacket extends InputPacket {
short rightStickX;
short rightStickY;
public MultiControllerPacket(short controllerNumber, short buttonFlags, byte leftTrigger, byte rightTrigger,
private int headerCode;
public MultiControllerPacket(ConnectionContext context,
short controllerNumber, short buttonFlags, byte leftTrigger, byte rightTrigger,
short leftStickX, short leftStickY,
short rightStickX, short rightStickY)
{
super(PACKET_TYPE);
this.headerCode = HEADER_CODE;
// On Gen 5 servers, the header code is decremented by one
if (context.serverGeneration >= ConnectionContext.SERVER_GENERATION_5) {
headerCode--;
}
this.controllerNumber = controllerNumber;
this.buttonFlags = buttonFlags;
@ -72,7 +85,7 @@ public class MultiControllerPacket extends InputPacket {
@Override
public void toWirePayload(ByteBuffer bb) {
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(0xd);
bb.putInt(headerCode);
bb.putShort((short) 0x1a);
bb.putShort(controllerNumber);
bb.putShort((short) 0x0f); // Active controller flags

View File

@ -40,8 +40,10 @@ public class RtspConnection {
case ConnectionContext.SERVER_GENERATION_3:
return 10;
case ConnectionContext.SERVER_GENERATION_4:
default:
return 11;
case ConnectionContext.SERVER_GENERATION_5:
default:
return 12;
}
}

View File

@ -53,37 +53,16 @@ public class SdpGenerator {
private static void addGen4Attributes(StringBuilder config, ConnectionContext context) {
addSessionAttribute(config, "x-nv-general.serverAddress", "rtsp://"+context.serverAddress.getHostAddress()+":48010");
// If client and server are able, request HEVC
if (context.negotiatedVideoFormat == VideoFormat.H265) {
addSessionAttribute(config, "x-nv-clientSupportHevc", "1");
addSessionAttribute(config, "x-nv-vqos[0].bitStreamFormat", "1");
// Disable slicing on HEVC
addSessionAttribute(config, "x-nv-video[0].videoEncoderSlicesPerFrame", "1");
}
else {
// Otherwise, use AVC
addSessionAttribute(config, "x-nv-clientSupportHevc", "0");
addSessionAttribute(config, "x-nv-vqos[0].bitStreamFormat", "0");
// Use slicing for increased performance on some decoders
addSessionAttribute(config, "x-nv-video[0].videoEncoderSlicesPerFrame", "4");
}
addSessionAttribute(config, "x-nv-video[0].rateControlMode", "4");
}
private static void addGen5Attributes(StringBuilder config, ConnectionContext context) {
// We want to use the legacy TCP connections for control and input rather than the new UDP stuff
addSessionAttribute(config, "x-nv-general.useReliableUdp", "0");
addSessionAttribute(config, "x-nv-ri.useControlChannel", "0");
// Enable surround sound if configured for it
addSessionAttribute(config, "x-nv-audio.surround.numChannels", ""+context.streamConfig.getAudioChannelCount());
addSessionAttribute(config, "x-nv-audio.surround.channelMask", ""+context.streamConfig.getAudioChannelMask());
if (context.streamConfig.getAudioChannelCount() > 2) {
addSessionAttribute(config, "x-nv-audio.surround.enable", "1");
}
else {
addSessionAttribute(config, "x-nv-audio.surround.enable", "0");
}
addSessionAttribute(config, "x-nv-vqos[0].enableQec", "0");
}
public static String generateSdpFromContext(ConnectionContext context) {
@ -116,11 +95,6 @@ public class SdpGenerator {
addSessionAttribute(config, "x-nv-video[0].packetSize", ""+context.streamConfig.getMaxPacketSize());
if (context.streamConfig.getRemote()) {
addSessionAttribute(config, "x-nv-video[0].averageBitrate", "4");
addSessionAttribute(config, "x-nv-video[0].peakBitrate", "4");
}
addSessionAttribute(config, "x-nv-video[0].timeoutLengthMs", "7000");
addSessionAttribute(config, "x-nv-video[0].framesWithInvalidRefThreshold", "0");
@ -135,11 +109,22 @@ public class SdpGenerator {
bitrate = context.streamConfig.getBitrate();
}
// We don't support dynamic bitrate scaling properly (it tends to bounce between min and max and never
// settle on the optimal bitrate if it's somewhere in the middle), so we'll just latch the bitrate
// to the requested value.
addSessionAttribute(config, "x-nv-vqos[0].bw.minimumBitrate", ""+bitrate);
addSessionAttribute(config, "x-nv-vqos[0].bw.maximumBitrate", ""+bitrate);
if (context.serverGeneration >= ConnectionContext.SERVER_GENERATION_5) {
addSessionAttribute(config, "x-nv-vqos[0].bw.minimumBitrateKbps", ""+bitrate);
addSessionAttribute(config, "x-nv-vqos[0].bw.maximumBitrateKbps", ""+bitrate);
}
else {
if (context.streamConfig.getRemote()) {
addSessionAttribute(config, "x-nv-video[0].averageBitrate", "4");
addSessionAttribute(config, "x-nv-video[0].peakBitrate", "4");
}
// We don't support dynamic bitrate scaling properly (it tends to bounce between min and max and never
// settle on the optimal bitrate if it's somewhere in the middle), so we'll just latch the bitrate
// to the requested value.
addSessionAttribute(config, "x-nv-vqos[0].bw.minimumBitrate", ""+bitrate);
addSessionAttribute(config, "x-nv-vqos[0].bw.maximumBitrate", ""+bitrate);
}
// Using FEC turns padding on which makes us have to take the slow path
// in the depacketizer, not to mention exposing some ambiguous cases with
@ -170,9 +155,42 @@ public class SdpGenerator {
break;
case ConnectionContext.SERVER_GENERATION_4:
default:
addGen4Attributes(config, context);
break;
case ConnectionContext.SERVER_GENERATION_5:
default:
addGen5Attributes(config, context);
break;
}
// Gen 4+ supports H.265 and surround sound
if (context.serverGeneration >= ConnectionContext.SERVER_GENERATION_4) {
// If client and server are able, request HEVC
if (context.negotiatedVideoFormat == VideoFormat.H265) {
addSessionAttribute(config, "x-nv-clientSupportHevc", "1");
addSessionAttribute(config, "x-nv-vqos[0].bitStreamFormat", "1");
// Disable slicing on HEVC
addSessionAttribute(config, "x-nv-video[0].videoEncoderSlicesPerFrame", "1");
}
else {
// Otherwise, use AVC
addSessionAttribute(config, "x-nv-clientSupportHevc", "0");
addSessionAttribute(config, "x-nv-vqos[0].bitStreamFormat", "0");
// Use slicing for increased performance on some decoders
addSessionAttribute(config, "x-nv-video[0].videoEncoderSlicesPerFrame", "4");
}
// Enable surround sound if configured for it
addSessionAttribute(config, "x-nv-audio.surround.numChannels", ""+context.streamConfig.getAudioChannelCount());
addSessionAttribute(config, "x-nv-audio.surround.channelMask", ""+context.streamConfig.getAudioChannelMask());
if (context.streamConfig.getAudioChannelCount() > 2) {
addSessionAttribute(config, "x-nv-audio.surround.enable", "1");
}
else {
addSessionAttribute(config, "x-nv-audio.surround.enable", "0");
}
}
config.append("t=0 0").append("\r\n");