diff --git a/moonlight-common/.classpath b/moonlight-common/.classpath new file mode 100644 index 00000000..62423bd8 --- /dev/null +++ b/moonlight-common/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/moonlight-common/.project b/moonlight-common/.project new file mode 100644 index 00000000..aff885f2 --- /dev/null +++ b/moonlight-common/.project @@ -0,0 +1,17 @@ + + + limelight-common + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/moonlight-common/.settings/org.eclipse.jdt.core.prefs b/moonlight-common/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..8000cd6c --- /dev/null +++ b/moonlight-common/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/moonlight-common/libs/xpp3-1.1.4c.jar b/moonlight-common/libs/xpp3-1.1.4c.jar new file mode 100644 index 00000000..451ac82a Binary files /dev/null and b/moonlight-common/libs/xpp3-1.1.4c.jar differ diff --git a/moonlight-common/src/com/limelight/nvstream/ControlStream.java b/moonlight-common/src/com/limelight/nvstream/ControlStream.java new file mode 100644 index 00000000..75bde0e4 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/ControlStream.java @@ -0,0 +1,505 @@ +package com.limelight.nvstream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import com.limelight.nvstream.av.ConnectionStatusListener; + +public class ControlStream implements ConnectionStatusListener { + + public static final int PORT = 47995; + + public static final int CONTROL_TIMEOUT = 5000; + + public static final short PTYPE_HELLO = 0x1204; + public static final short PPAYLEN_HELLO = 0x0004; + public static final byte[] PPAYLOAD_HELLO = + { + (byte)0x00, + (byte)0x05, + (byte)0x00, + (byte)0x00 + }; + + public static final short PTYPE_KEEPALIVE = 0x13ff; + public static final short PPAYLEN_KEEPALIVE = 0x0000; + + public static final short PTYPE_HEARTBEAT = 0x1401; + public static final short PPAYLEN_HEARTBEAT = 0x0000; + + public static final short PTYPE_1405 = 0x1405; + public static final short PPAYLEN_1405 = 0x0000; + + public static final short PTYPE_RESYNC = 0x1404; + public static final short PPAYLEN_RESYNC = 16; + + public static final short PTYPE_CONFIG = 0x1205; + public static final short PPAYLEN_CONFIG = 0x0004; + public static final int[] PPAYLOAD_CONFIG = + { + 720, + 266758, + 1, + 266762, + 30, + 70151, + 68291329, + 1280, + 68291584, + 1280, + 68291840, + 15360, + 68292096, + 25600, + 68292352, + 2048, + 68292608, + 1024, + 68289024, + 262144, + 17957632, + 302055424, + 134217729, + 16777490, + 70153, + 68293120, + 768000, + 17961216, + 303235072, + 335609857, + 838861842, + 352321536, + 1006634002, + 369098752, + 335545362, + 385875968, + 1042, + 402653184, + 134218770, + 419430400, + 167773202, + 436207616, + 855638290, + 266779, + 7000, + 266780, + 2000, + 266781, + 50, + 266782, + 3000, + 266783, + 2, + 266794, + 5000, + 266795, + 500, + 266784, + 75, + 266785, + 25, + 266786, + 10, + 266787, + 60, + 266788, + 30, + 266789, + 3, + 266790, + 1000, + 266791, + 5000, + 266792, + 5000, + 266793, + 5000, + 70190, + 68301063, + 10240, + 68301312, + 6400, + 68301568, + 768000, + 68299776, + 768, + 68300032, + 2560, + 68300544, + 0, + 34746368, + (int)0xFE000000 + }; + + + public static final short PTYPE_JITTER = 0x140c; + public static final short PPAYLEN_JITTER = 0x10; + + private int seqNum; + + private NvConnectionListener listener; + private InetAddress host; + + private Socket s; + private InputStream in; + private OutputStream out; + + private Thread heartbeatThread; + private Thread jitterThread; + private Thread resyncThread; + private Object resyncNeeded = new Object(); + private boolean aborting = false; + + public ControlStream(InetAddress host, NvConnectionListener listener) + { + this.listener = listener; + this.host = host; + } + + public void initialize() throws IOException + { + s = new Socket(); + s.setSoTimeout(CONTROL_TIMEOUT); + s.setTcpNoDelay(true); + s.connect(new InetSocketAddress(host, PORT), CONTROL_TIMEOUT); + in = s.getInputStream(); + out = s.getOutputStream(); + } + + private void sendPacket(NvCtlPacket packet) throws IOException + { + out.write(packet.toWire()); + out.flush(); + } + + private ControlStream.NvCtlResponse sendAndGetReply(NvCtlPacket packet) throws IOException + { + sendPacket(packet); + return new NvCtlResponse(in); + } + + private void sendJitter() throws IOException + { + ByteBuffer bb = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + + bb.putInt(0); + bb.putInt(77); + bb.putInt(888); + bb.putInt(seqNum += 2); + + sendPacket(new NvCtlPacket(PTYPE_JITTER, PPAYLEN_JITTER, bb.array())); + } + + public void abort() + { + if (aborting) { + return; + } + + aborting = true; + + if (jitterThread != null) { + jitterThread.interrupt(); + } + + if (heartbeatThread != null) { + heartbeatThread.interrupt(); + } + + try { + s.close(); + } catch (IOException e) {} + } + + public void requestResync() throws IOException + { + System.out.println("CTL: Requesting IDR frame"); + sendResync(); + } + + public void start() throws IOException + { + sendHello(); + sendConfig(); + pingPong(); + send1405AndGetResponse(); + + heartbeatThread = new Thread() { + @Override + public void run() { + while (!isInterrupted()) + { + try { + sendHeartbeat(); + } catch (IOException e) { + listener.connectionTerminated(e); + return; + } + + + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + listener.connectionTerminated(e); + return; + } + } + } + }; + heartbeatThread.start(); + + resyncThread = new Thread() { + @Override + public void run() { + while (!isInterrupted()) + { + try { + // Wait for notification of a resync needed + synchronized (resyncNeeded) { + resyncNeeded.wait(); + } + } catch (InterruptedException e) { + listener.connectionTerminated(e); + return; + } + + try { + requestResync(); + } catch (IOException e) { + listener.connectionTerminated(e); + return; + } + } + } + }; + resyncThread.start(); + } + + public void startJitterPackets() + { + jitterThread = new Thread() { + @Override + public void run() { + while (!isInterrupted()) + { + try { + sendJitter(); + } catch (IOException e) { + listener.connectionTerminated(e); + return; + } + + try { + Thread.sleep(100); + } catch (InterruptedException e) { + listener.connectionTerminated(e); + return; + } + } + } + }; + jitterThread.start(); + } + + private ControlStream.NvCtlResponse send1405AndGetResponse() throws IOException + { + return sendAndGetReply(new NvCtlPacket(PTYPE_1405, PPAYLEN_1405)); + } + + private void sendHello() throws IOException + { + sendPacket(new NvCtlPacket(PTYPE_HELLO, PPAYLEN_HELLO, PPAYLOAD_HELLO)); + } + + private void sendResync() throws IOException + { + ByteBuffer conf = ByteBuffer.wrap(new byte[PPAYLEN_RESYNC]).order(ByteOrder.LITTLE_ENDIAN); + + conf.putLong(0); + conf.putLong(0xFFFF); + + sendAndGetReply(new NvCtlPacket(PTYPE_RESYNC, PPAYLEN_RESYNC, conf.array())); + } + + private void sendConfig() throws IOException + { + ByteBuffer conf = ByteBuffer.wrap(new byte[PPAYLOAD_CONFIG.length * 4 + 3]).order(ByteOrder.LITTLE_ENDIAN); + + for (int i : PPAYLOAD_CONFIG) + conf.putInt(i); + + conf.putShort((short)0x0013); + conf.put((byte) 0x00); + + sendPacket(new NvCtlPacket(PTYPE_CONFIG, PPAYLEN_CONFIG, conf.array())); + } + + private void sendHeartbeat() throws IOException + { + sendPacket(new NvCtlPacket(PTYPE_HEARTBEAT, PPAYLEN_HEARTBEAT)); + } + + private ControlStream.NvCtlResponse pingPong() throws IOException + { + sendPacket(new NvCtlPacket(PTYPE_KEEPALIVE, PPAYLEN_KEEPALIVE)); + return new ControlStream.NvCtlResponse(in); + } + + class NvCtlPacket { + public short type; + public short paylen; + public byte[] payload; + + public NvCtlPacket(InputStream in) throws IOException + { + byte[] header = new byte[4]; + + int offset = 0; + do + { + int bytesRead = in.read(header, offset, header.length - offset); + if (bytesRead < 0) { + break; + } + offset += bytesRead; + } while (offset != header.length); + + if (offset != header.length) { + throw new IOException("Socket closed prematurely"); + } + + ByteBuffer bb = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + + type = bb.getShort(); + paylen = bb.getShort(); + + if (paylen != 0) + { + payload = new byte[paylen]; + + offset = 0; + do + { + int bytesRead = in.read(payload, offset, payload.length - offset); + if (bytesRead < 0) { + break; + } + offset += bytesRead; + } while (offset != payload.length); + + if (offset != payload.length) { + throw new IOException("Socket closed prematurely"); + } + } + } + + public NvCtlPacket(byte[] payload) + { + ByteBuffer bb = ByteBuffer.wrap(payload).order(ByteOrder.LITTLE_ENDIAN); + + type = bb.getShort(); + paylen = bb.getShort(); + + if (bb.hasRemaining()) + { + payload = new byte[bb.remaining()]; + bb.get(payload); + } + } + + public NvCtlPacket(short type, short paylen) + { + this.type = type; + this.paylen = paylen; + } + + public NvCtlPacket(short type, short paylen, byte[] payload) + { + this.type = type; + this.paylen = paylen; + this.payload = payload; + } + + public short getType() + { + return type; + } + + public short getPaylen() + { + return paylen; + } + + public void setType(short type) + { + this.type = type; + } + + public void setPaylen(short paylen) + { + this.paylen = paylen; + } + + public byte[] toWire() + { + ByteBuffer bb = ByteBuffer.allocate(4 + (payload != null ? payload.length : 0)).order(ByteOrder.LITTLE_ENDIAN); + + bb.putShort(type); + bb.putShort(paylen); + + if (payload != null) + bb.put(payload); + + return bb.array(); + } + } + + class NvCtlResponse extends NvCtlPacket { + public short status; + + public NvCtlResponse(InputStream in) throws IOException { + super(in); + } + + public NvCtlResponse(short type, short paylen) { + super(type, paylen); + } + + public NvCtlResponse(short type, short paylen, byte[] payload) { + super(type, paylen, payload); + } + + public NvCtlResponse(byte[] payload) { + super(payload); + } + + public void setStatusCode(short status) + { + this.status = status; + } + + public short getStatusCode() + { + return status; + } + } + + @Override + public void connectionTerminated() { + abort(); + } + + @Override + public void connectionNeedsResync() { + synchronized (resyncNeeded) { + // Wake up the resync thread + resyncNeeded.notify(); + } + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/Handshake.java b/moonlight-common/src/com/limelight/nvstream/Handshake.java new file mode 100644 index 00000000..e3249af9 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/Handshake.java @@ -0,0 +1,133 @@ +package com.limelight.nvstream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; + +public class Handshake { + public static final int PORT = 47991; + + public static final int HANDSHAKE_TIMEOUT = 5000; + + public static final byte[] PLATFORM_HELLO = + { + (byte)0x07, + (byte)0x00, + (byte)0x00, + (byte)0x00, + + // android in ASCII + (byte)0x61, + (byte)0x6e, + (byte)0x64, + (byte)0x72, + (byte)0x6f, + (byte)0x69, + (byte)0x64, + + (byte)0x03, + (byte)0x01, + (byte)0x00, + (byte)0x00 + }; + + public static final byte[] PACKET_2 = + { + (byte)0x01, + (byte)0x03, + (byte)0x02, + (byte)0x00, + (byte)0x08, + (byte)0x00 + }; + + public static final byte[] PACKET_3 = + { + (byte)0x04, + (byte)0x01, + (byte)0x00, + (byte)0x00, + + (byte)0x00, + (byte)0x00, + (byte)0x00, + (byte)0x00 + }; + + public static final byte[] PACKET_4 = + { + (byte)0x01, + (byte)0x01, + (byte)0x00, + (byte)0x00 + }; + + private static boolean waitAndDiscardResponse(InputStream in) + { + // Wait for response and discard response + try { + in.read(); + + // Wait for the full response to come in + Thread.sleep(250); + + for (int i = 0; i < in.available(); i++) + in.read(); + + } catch (IOException e1) { + return false; + } catch (InterruptedException e) { + return false; + } + + return true; + } + + public static boolean performHandshake(InetAddress host) throws IOException + { + Socket s = new Socket(); + s.connect(new InetSocketAddress(host, PORT), HANDSHAKE_TIMEOUT); + s.setSoTimeout(HANDSHAKE_TIMEOUT); + OutputStream out = s.getOutputStream(); + InputStream in = s.getInputStream(); + + // First packet + out.write(PLATFORM_HELLO); + out.flush(); + + if (!waitAndDiscardResponse(in)) { + s.close(); + return false; + } + + // Second packet + out.write(PACKET_2); + out.flush(); + + if (!waitAndDiscardResponse(in)) { + s.close(); + return false; + } + + // Third packet + out.write(PACKET_3); + out.flush(); + + if (!waitAndDiscardResponse(in)) { + s.close(); + return false; + } + + // Fourth packet + out.write(PACKET_4); + out.flush(); + + // Done + s.close(); + + return true; + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/NvConnection.java b/moonlight-common/src/com/limelight/nvstream/NvConnection.java new file mode 100644 index 00000000..54ed4f3b --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/NvConnection.java @@ -0,0 +1,321 @@ +package com.limelight.nvstream; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.Enumeration; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.xmlpull.v1.XmlPullParserException; + +import com.limelight.nvstream.av.audio.AudioStream; +import com.limelight.nvstream.av.audio.AudioRenderer; +import com.limelight.nvstream.av.video.VideoDecoderRenderer; +import com.limelight.nvstream.av.video.VideoStream; +import com.limelight.nvstream.http.NvHTTP; +import com.limelight.nvstream.input.NvController; + +public class NvConnection { + private String host; + private NvConnectionListener listener; + + private InetAddress hostAddr; + private ControlStream controlStream; + private NvController inputStream; + private VideoStream videoStream; + private AudioStream audioStream; + + // Start parameters + private int drFlags; + private Object videoRenderTarget; + private VideoDecoderRenderer videoDecoderRenderer; + private AudioRenderer audioRenderer; + private String localDeviceName; + + private ThreadPoolExecutor threadPool; + + public NvConnection(String host, NvConnectionListener listener) + { + this.host = host; + this.listener = listener; + this.threadPool = new ThreadPoolExecutor(1, 1, Long.MAX_VALUE, TimeUnit.DAYS, new LinkedBlockingQueue()); + } + + public static String getMacAddressString() throws SocketException { + Enumeration ifaceList; + NetworkInterface selectedIface = null; + + // First look for a WLAN interface (since those generally aren't removable) + ifaceList = NetworkInterface.getNetworkInterfaces(); + while (selectedIface == null && ifaceList.hasMoreElements()) { + NetworkInterface iface = ifaceList.nextElement(); + + if (iface.getName().startsWith("wlan") && + iface.getHardwareAddress() != null) { + selectedIface = iface; + } + } + + // If we didn't find that, look for an Ethernet interface + ifaceList = NetworkInterface.getNetworkInterfaces(); + while (selectedIface == null && ifaceList.hasMoreElements()) { + NetworkInterface iface = ifaceList.nextElement(); + + if (iface.getName().startsWith("eth") && + iface.getHardwareAddress() != null) { + selectedIface = iface; + } + } + + // Now just find something with a MAC address + ifaceList = NetworkInterface.getNetworkInterfaces(); + while (selectedIface == null && ifaceList.hasMoreElements()) { + NetworkInterface iface = ifaceList.nextElement(); + + if (iface.getHardwareAddress() != null) { + selectedIface = ifaceList.nextElement(); + break; + } + } + + if (selectedIface == null) { + return null; + } + + byte[] macAddress = selectedIface.getHardwareAddress(); + if (macAddress != null) { + StringBuilder addrStr = new StringBuilder(); + for (int i = 0; i < macAddress.length; i++) { + addrStr.append(String.format("%02x", macAddress[i])); + if (i != macAddress.length - 1) { + addrStr.append(':'); + } + } + return addrStr.toString(); + } + + return null; + } + + public void stop() + { + threadPool.shutdownNow(); + + if (videoStream != null) { + videoStream.abort(); + } + if (audioStream != null) { + audioStream.abort(); + } + + if (controlStream != null) { + controlStream.abort(); + } + + if (inputStream != null) { + inputStream.close(); + inputStream = null; + } + } + + private boolean startSteamBigPicture() throws XmlPullParserException, IOException + { + NvHTTP h = new NvHTTP(hostAddr, getMacAddressString(), localDeviceName); + + if (!h.getPairState()) { + listener.displayMessage("Device not paired with computer"); + return false; + } + + int sessionId = h.getSessionId(); + int appId = h.getSteamAppId(sessionId); + + h.launchApp(sessionId, appId); + + return true; + } + + private boolean startControlStream() throws IOException + { + controlStream = new ControlStream(hostAddr, listener); + controlStream.initialize(); + controlStream.start(); + return true; + } + + private boolean startVideoStream() throws IOException + { + videoStream = new VideoStream(hostAddr, listener, controlStream); + videoStream.startVideoStream(videoDecoderRenderer, videoRenderTarget, drFlags); + return true; + } + + private boolean startAudioStream() throws IOException + { + audioStream = new AudioStream(hostAddr, listener, audioRenderer); + audioStream.startAudioStream(); + return true; + } + + private boolean startInputConnection() throws IOException + { + inputStream = new NvController(hostAddr); + inputStream.initialize(); + return true; + } + + private void establishConnection() { + for (NvConnectionListener.Stage currentStage : NvConnectionListener.Stage.values()) + { + boolean success = false; + + listener.stageStarting(currentStage); + try { + switch (currentStage) + { + case LAUNCH_APP: + success = startSteamBigPicture(); + break; + + case HANDSHAKE: + success = Handshake.performHandshake(hostAddr); + break; + + case CONTROL_START: + success = startControlStream(); + break; + + case VIDEO_START: + success = startVideoStream(); + break; + + case AUDIO_START: + success = startAudioStream(); + break; + + case CONTROL_START2: + controlStream.startJitterPackets(); + success = true; + break; + + case INPUT_START: + success = startInputConnection(); + break; + } + } catch (Exception e) { + e.printStackTrace(); + success = false; + } + + if (success) { + listener.stageComplete(currentStage); + } + else { + listener.stageFailed(currentStage); + return; + } + } + + listener.connectionStarted(); + } + + public void start(String localDeviceName, Object videoRenderTarget, int drFlags, AudioRenderer audioRenderer, VideoDecoderRenderer videoDecoderRenderer) + { + this.localDeviceName = localDeviceName; + this.drFlags = drFlags; + this.audioRenderer = audioRenderer; + this.videoRenderTarget = videoRenderTarget; + this.videoDecoderRenderer = videoDecoderRenderer; + + new Thread(new Runnable() { + @Override + public void run() { + try { + hostAddr = InetAddress.getByName(host); + } catch (UnknownHostException e) { + listener.connectionTerminated(e); + return; + } + + establishConnection(); + } + }).start(); + } + + public void sendMouseMove(final short deltaX, final short deltaY) + { + if (inputStream == null) + return; + + threadPool.execute(new Runnable() { + @Override + public void run() { + try { + inputStream.sendMouseMove(deltaX, deltaY); + } catch (IOException e) { + listener.connectionTerminated(e); + } + } + }); + } + + public void sendMouseButtonDown() + { + if (inputStream == null) + return; + + threadPool.execute(new Runnable() { + @Override + public void run() { + try { + inputStream.sendMouseButtonDown(); + } catch (IOException e) { + listener.connectionTerminated(e); + } + } + }); + } + + public void sendMouseButtonUp() + { + if (inputStream == null) + return; + + threadPool.execute(new Runnable() { + @Override + public void run() { + try { + inputStream.sendMouseButtonUp(); + } catch (IOException e) { + listener.connectionTerminated(e); + } + } + }); + } + + public void sendControllerInput(final short buttonFlags, + final byte leftTrigger, final byte rightTrigger, + final short leftStickX, final short leftStickY, + final short rightStickX, final short rightStickY) + { + if (inputStream == null) + return; + + threadPool.execute(new Runnable() { + @Override + public void run() { + try { + inputStream.sendControllerInput(buttonFlags, leftTrigger, + rightTrigger, leftStickX, leftStickY, + rightStickX, rightStickY); + } catch (IOException e) { + listener.connectionTerminated(e); + } + } + }); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/NvConnectionListener.java b/moonlight-common/src/com/limelight/nvstream/NvConnectionListener.java new file mode 100644 index 00000000..0fb63dc2 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/NvConnectionListener.java @@ -0,0 +1,32 @@ +package com.limelight.nvstream; + +public interface NvConnectionListener { + + public enum Stage { + LAUNCH_APP("app"), + HANDSHAKE("handshake"), + CONTROL_START("control connection"), + VIDEO_START("video stream"), + AUDIO_START("audio stream"), + CONTROL_START2("control connection"), + INPUT_START("input connection"); + + private String name; + private Stage(String name) { + this.name = name; + } + + public String getName() { + return name; + } + }; + + public void stageStarting(Stage stage); + public void stageComplete(Stage stage); + public void stageFailed(Stage stage); + + public void connectionStarted(); + public void connectionTerminated(Exception e); + + public void displayMessage(String message); +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/ByteBufferDescriptor.java b/moonlight-common/src/com/limelight/nvstream/av/ByteBufferDescriptor.java new file mode 100644 index 00000000..83fa0b46 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/ByteBufferDescriptor.java @@ -0,0 +1,46 @@ +package com.limelight.nvstream.av; + +public class ByteBufferDescriptor { + public byte[] data; + public int offset; + public int length; + + public ByteBufferDescriptor(byte[] data, int offset, int length) + { + this.data = data; + this.offset = offset; + this.length = length; + } + + public ByteBufferDescriptor(ByteBufferDescriptor desc) + { + this.data = desc.data; + this.offset = desc.offset; + this.length = desc.length; + } + + public void reinitialize(byte[] data, int offset, int length) + { + this.data = data; + this.offset = offset; + this.length = length; + } + + public void print() + { + print(offset, length); + } + + public void print(int length) + { + print(this.offset, length); + } + + public void print(int offset, int length) + { + for (int i = offset; i < offset+length; i++) { + System.out.printf("%d: %02x \n", i, data[i]); + } + System.out.println(); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/ConnectionStatusListener.java b/moonlight-common/src/com/limelight/nvstream/av/ConnectionStatusListener.java new file mode 100644 index 00000000..35262ddf --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/ConnectionStatusListener.java @@ -0,0 +1,7 @@ +package com.limelight.nvstream.av; + +public interface ConnectionStatusListener { + public void connectionTerminated(); + + public void connectionNeedsResync(); +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/DecodeUnit.java b/moonlight-common/src/com/limelight/nvstream/av/DecodeUnit.java new file mode 100644 index 00000000..f58510e0 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/DecodeUnit.java @@ -0,0 +1,42 @@ +package com.limelight.nvstream.av; + +import java.util.List; + +public class DecodeUnit { + public static final int TYPE_UNKNOWN = 0; + public static final int TYPE_H264 = 1; + public static final int TYPE_OPUS = 2; + + private int type; + private List bufferList; + private int dataLength; + private int flags; + + public DecodeUnit(int type, List bufferList, int dataLength, int flags) + { + this.type = type; + this.bufferList = bufferList; + this.dataLength = dataLength; + this.flags = flags; + } + + public int getType() + { + return type; + } + + public int getFlags() + { + return flags; + } + + public List getBufferList() + { + return bufferList; + } + + public int getDataLength() + { + return dataLength; + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/RtpPacket.java b/moonlight-common/src/com/limelight/nvstream/av/RtpPacket.java new file mode 100644 index 00000000..d1fc429e --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/RtpPacket.java @@ -0,0 +1,46 @@ +package com.limelight.nvstream.av; + +import java.nio.ByteBuffer; + +public class RtpPacket { + + private byte packetType; + private short seqNum; + private ByteBufferDescriptor buffer; + + public RtpPacket(ByteBufferDescriptor buffer) + { + this.buffer = new ByteBufferDescriptor(buffer); + + ByteBuffer bb = ByteBuffer.wrap(buffer.data, buffer.offset, buffer.length); + + // Discard the first byte + bb.position(bb.position()+1); + + // Get the packet type + packetType = bb.get(); + + // Get the sequence number + seqNum = bb.getShort(); + } + + public byte getPacketType() + { + return packetType; + } + + public short getSequenceNumber() + { + return seqNum; + } + + public byte[] getBackingBuffer() + { + return buffer.data; + } + + public ByteBufferDescriptor getNewPayloadDescriptor() + { + return new ByteBufferDescriptor(buffer.data, buffer.offset+12, buffer.length-12); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/ShortBufferDescriptor.java b/moonlight-common/src/com/limelight/nvstream/av/ShortBufferDescriptor.java new file mode 100644 index 00000000..44cd2d8e --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/ShortBufferDescriptor.java @@ -0,0 +1,28 @@ +package com.limelight.nvstream.av; + +public class ShortBufferDescriptor { + public short[] data; + public int offset; + public int length; + + public ShortBufferDescriptor(short[] data, int offset, int length) + { + this.data = data; + this.offset = offset; + this.length = length; + } + + public ShortBufferDescriptor(ShortBufferDescriptor desc) + { + this.data = desc.data; + this.offset = desc.offset; + this.length = desc.length; + } + + public void reinitialize(short[] data, int offset, int length) + { + this.data = data; + this.offset = offset; + this.length = length; + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioDepacketizer.java b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioDepacketizer.java new file mode 100644 index 00000000..c8fd728d --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioDepacketizer.java @@ -0,0 +1,65 @@ +package com.limelight.nvstream.av.audio; + +import java.util.concurrent.LinkedBlockingQueue; + +import com.limelight.nvstream.av.ByteBufferDescriptor; +import com.limelight.nvstream.av.RtpPacket; +import com.limelight.nvstream.av.ShortBufferDescriptor; + +public class AudioDepacketizer { + + private static final int DU_LIMIT = 15; + private LinkedBlockingQueue decodedUnits = + new LinkedBlockingQueue(DU_LIMIT); + + // Sequencing state + private short lastSequenceNumber; + + private void decodeData(byte[] data, int off, int len) + { + // Submit this data to the decoder + short[] pcmData = new short[OpusDecoder.getMaxOutputShorts()]; + int decodeLen = OpusDecoder.decode(data, off, len, pcmData); + + if (decodeLen > 0) { + // Return value of decode is frames decoded per channel + decodeLen *= OpusDecoder.getChannelCount(); + + // Put it on the decoded queue + if (!decodedUnits.offer(new ShortBufferDescriptor(pcmData, 0, decodeLen))) { + // Clear out the queue + decodedUnits.clear(); + } + } + } + + public void decodeInputData(RtpPacket packet) + { + short seq = packet.getSequenceNumber(); + + if (packet.getPacketType() != 97) { + // Only type 97 is audio + return; + } + + // Toss out the current NAL if we receive a packet that is + // out of sequence + if (lastSequenceNumber != 0 && + (short)(lastSequenceNumber + 1) != seq) + { + System.out.println("Received OOS audio data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); + decodeData(null, 0, 0); + } + + lastSequenceNumber = seq; + + // This is all the depacketizing we need to do + ByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); + decodeData(rtpPayload.data, rtpPayload.offset, rtpPayload.length); + } + + public ShortBufferDescriptor getNextDecodedData() throws InterruptedException + { + return decodedUnits.take(); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioRenderer.java b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioRenderer.java new file mode 100644 index 00000000..ebffda58 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioRenderer.java @@ -0,0 +1,9 @@ +package com.limelight.nvstream.av.audio; + +public interface AudioRenderer { + public void streamInitialized(int channelCount, int sampleRate); + + public void playDecodedAudio(short[] audioData, int offset, int length); + + public void streamClosing(); +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/audio/AudioStream.java b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioStream.java new file mode 100644 index 00000000..19a2a1cb --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/audio/AudioStream.java @@ -0,0 +1,223 @@ +package com.limelight.nvstream.av.audio; + +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketException; +import java.util.LinkedList; +import java.util.concurrent.LinkedBlockingQueue; + +import com.limelight.nvstream.NvConnectionListener; +import com.limelight.nvstream.av.ByteBufferDescriptor; +import com.limelight.nvstream.av.RtpPacket; +import com.limelight.nvstream.av.ShortBufferDescriptor; + +public class AudioStream { + public static final int RTP_PORT = 48000; + public static final int RTCP_PORT = 47999; + + private LinkedBlockingQueue packets = new LinkedBlockingQueue(100); + + private DatagramSocket rtp; + + private AudioDepacketizer depacketizer = new AudioDepacketizer(); + + private LinkedList threads = new LinkedList(); + + private boolean aborting = false; + + private InetAddress host; + private NvConnectionListener connListener; + private AudioRenderer streamListener; + + public AudioStream(InetAddress host, NvConnectionListener connListener, AudioRenderer streamListener) + { + this.host = host; + this.connListener = connListener; + this.streamListener = streamListener; + } + + public void abort() + { + if (aborting) { + return; + } + + aborting = true; + + for (Thread t : threads) { + t.interrupt(); + } + + // Close the socket to interrupt the receive thread + if (rtp != null) { + rtp.close(); + } + + // Wait for threads to terminate + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { } + } + + streamListener.streamClosing(); + + threads.clear(); + } + + public void startAudioStream() throws SocketException + { + setupRtpSession(); + + setupAudio(); + + startReceiveThread(); + + startDepacketizerThread(); + + startDecoderThread(); + + startUdpPingThread(); + } + + private void setupRtpSession() throws SocketException + { + rtp = new DatagramSocket(RTP_PORT); + } + + private void setupAudio() + { + int err; + + err = OpusDecoder.init(); + if (err != 0) { + throw new IllegalStateException("Opus decoder failed to initialize"); + } + + streamListener.streamInitialized(OpusDecoder.getChannelCount(), OpusDecoder.getSampleRate()); + } + + private void startDepacketizerThread() + { + // This thread lessens the work on the receive thread + // so it can spend more time waiting for data + Thread t = new Thread() { + @Override + public void run() { + while (!isInterrupted()) + { + RtpPacket packet; + + try { + packet = packets.take(); + } catch (InterruptedException e) { + connListener.connectionTerminated(e); + return; + } + + depacketizer.decodeInputData(packet); + } + } + }; + threads.add(t); + t.setName("Audio - Depacketizer"); + t.start(); + } + + private void startDecoderThread() + { + // Decoder thread + Thread t = new Thread() { + @Override + public void run() { + while (!isInterrupted()) + { + ShortBufferDescriptor samples; + + try { + samples = depacketizer.getNextDecodedData(); + } catch (InterruptedException e) { + connListener.connectionTerminated(e); + return; + } + + streamListener.playDecodedAudio(samples.data, samples.offset, samples.length); + } + } + }; + threads.add(t); + t.setName("Audio - Player"); + t.start(); + } + + private void startReceiveThread() + { + // Receive thread + Thread t = new Thread() { + @Override + public void run() { + ByteBufferDescriptor desc = new ByteBufferDescriptor(new byte[1500], 0, 1500); + DatagramPacket packet = new DatagramPacket(desc.data, desc.length); + + while (!isInterrupted()) + { + try { + rtp.receive(packet); + } catch (IOException e) { + connListener.connectionTerminated(e); + return; + } + + // Give the packet to the depacketizer thread + desc.length = packet.getLength(); + if (packets.offer(new RtpPacket(desc))) { + desc.reinitialize(new byte[1500], 0, 1500); + packet.setData(desc.data, desc.offset, desc.length); + } + } + } + }; + threads.add(t); + t.setName("Audio - Receive"); + t.start(); + } + + private void startUdpPingThread() + { + // Ping thread + Thread t = new Thread() { + @Override + public void run() { + // PING in ASCII + final byte[] pingPacketData = new byte[] {0x50, 0x49, 0x4E, 0x47}; + DatagramPacket pingPacket = new DatagramPacket(pingPacketData, pingPacketData.length); + pingPacket.setSocketAddress(new InetSocketAddress(host, RTP_PORT)); + + // Send PING every 100 ms + while (!isInterrupted()) + { + try { + rtp.send(pingPacket); + } catch (IOException e) { + connListener.connectionTerminated(e); + return; + } + + try { + Thread.sleep(100); + } catch (InterruptedException e) { + connListener.connectionTerminated(e); + return; + } + } + } + }; + threads.add(t); + t.setPriority(Thread.MIN_PRIORITY); + t.setName("Audio - Ping"); + t.start(); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/audio/OpusDecoder.java b/moonlight-common/src/com/limelight/nvstream/av/audio/OpusDecoder.java new file mode 100644 index 00000000..c01f7fa5 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/audio/OpusDecoder.java @@ -0,0 +1,14 @@ +package com.limelight.nvstream.av.audio; + +public class OpusDecoder { + static { + System.loadLibrary("nv_opus_dec"); + } + + public static native int init(); + public static native void destroy(); + public static native int getChannelCount(); + public static native int getMaxOutputShorts(); + public static native int getSampleRate(); + public static native int decode(byte[] indata, int inoff, int inlen, short[] outpcmdata); +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java new file mode 100644 index 00000000..733d23fd --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDecoderRenderer.java @@ -0,0 +1,17 @@ +package com.limelight.nvstream.av.video; + +import com.limelight.nvstream.av.DecodeUnit; + +public interface VideoDecoderRenderer { + public static int FLAG_PREFER_QUALITY = 0x1; + + public void setup(int width, int height, Object renderTarget, int drFlags); + + public void start(); + + public void stop(); + + public void release(); + + public boolean submitDecodeUnit(DecodeUnit decodeUnit); +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java new file mode 100644 index 00000000..d1936a34 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoDepacketizer.java @@ -0,0 +1,213 @@ +package com.limelight.nvstream.av.video; + +import java.util.LinkedList; +import java.util.concurrent.LinkedBlockingQueue; + +import com.limelight.nvstream.av.ByteBufferDescriptor; +import com.limelight.nvstream.av.DecodeUnit; +import com.limelight.nvstream.av.RtpPacket; +import com.limelight.nvstream.av.ConnectionStatusListener; + +public class VideoDepacketizer { + + // Current NAL state + private LinkedList avcNalDataChain = null; + private int avcNalDataLength = 0; + + // Sequencing state + private short lastSequenceNumber; + + private ConnectionStatusListener controlListener; + + private static final int DU_LIMIT = 15; + private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(DU_LIMIT); + + public VideoDepacketizer(ConnectionStatusListener controlListener) + { + this.controlListener = controlListener; + } + + private void clearAvcNalState() + { + avcNalDataChain = null; + avcNalDataLength = 0; + } + + private void reassembleAvcNal() + { + // This is the start of a new NAL + if (avcNalDataChain != null && avcNalDataLength != 0) { + // Construct the H264 decode unit + DecodeUnit du = new DecodeUnit(DecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, 0); + if (!decodedUnits.offer(du)) { + // We need a new IDR frame since we're discarding data now + decodedUnits.clear(); + controlListener.connectionNeedsResync(); + } + + // Clear old state + clearAvcNalState(); + } + } + + public void addInputData(VideoPacket packet) + { + ByteBufferDescriptor location = packet.getNewPayloadDescriptor(); + + // SPS and PPS packet doesn't have standard headers, so submit it as is + if (location.length < 968) { + avcNalDataChain = new LinkedList(); + avcNalDataLength = 0; + + avcNalDataChain.add(location); + avcNalDataLength += location.length; + + reassembleAvcNal(); + } + else { + int packetIndex = packet.getPacketIndex(); + int packetsInFrame = packet.getTotalPackets(); + + // Check if this is the first packet for a frame + if (packetIndex == 0) { + // Setup state for the new frame + avcNalDataChain = new LinkedList(); + avcNalDataLength = 0; + } + + // Check if this packet falls in the range of packets in frame + if (packetIndex >= packetsInFrame) { + // This isn't H264 frame data + return; + } + + // Adjust the length to only contain valid data + location.length = packet.getPayloadLength(); + + // Add the payload data to the chain + if (avcNalDataChain != null) { + avcNalDataChain.add(location); + avcNalDataLength += location.length; + } + + // Reassemble the NALs if this was the last packet for this frame + if (packetIndex + 1 == packetsInFrame) { + reassembleAvcNal(); + } + } + } + + public void addInputData(RtpPacket packet) + { + short seq = packet.getSequenceNumber(); + + // Toss out the current NAL if we receive a packet that is + // out of sequence + if (lastSequenceNumber != 0 && + (short)(lastSequenceNumber + 1) != seq) + { + System.out.println("Received OOS video data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); + + // Reset the depacketizer state + clearAvcNalState(); + + // Request an IDR frame + controlListener.connectionNeedsResync(); + } + + lastSequenceNumber = seq; + + // Pass the payload to the non-sequencing parser + ByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); + addInputData(new VideoPacket(rtpPayload)); + } + + public DecodeUnit getNextDecodeUnit() throws InterruptedException + { + return decodedUnits.take(); + } +} + +class NAL { + + // This assumes that the buffer passed in is already a special sequence + public static boolean isAvcStartSequence(ByteBufferDescriptor specialSeq) + { + // The start sequence is 00 00 01 or 00 00 00 01 + return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01); + } + + // This assumes that the buffer passed in is already a special sequence + public static boolean isAvcFrameStart(ByteBufferDescriptor specialSeq) + { + if (specialSeq.length != 4) + return false; + + // The frame start sequence is 00 00 00 01 + return (specialSeq.data[specialSeq.offset+specialSeq.length-1] == 0x01); + } + + // Returns a buffer descriptor describing the start sequence + public static boolean getSpecialSequenceDescriptor(ByteBufferDescriptor buffer, ByteBufferDescriptor outputDesc) + { + // NAL start sequence is 00 00 00 01 or 00 00 01 + if (buffer.length < 3) + return false; + + // 00 00 is magic + if (buffer.data[buffer.offset] == 0x00 && + buffer.data[buffer.offset+1] == 0x00) + { + // Another 00 could be the end of the special sequence + // 00 00 00 or the middle of 00 00 00 01 + if (buffer.data[buffer.offset+2] == 0x00) + { + if (buffer.length >= 4 && + buffer.data[buffer.offset+3] == 0x01) + { + // It's the AVC start sequence 00 00 00 01 + outputDesc.reinitialize(buffer.data, buffer.offset, 4); + } + else + { + // It's 00 00 00 + outputDesc.reinitialize(buffer.data, buffer.offset, 3); + } + return true; + } + else if (buffer.data[buffer.offset+2] == 0x01 || + buffer.data[buffer.offset+2] == 0x02) + { + // These are easy: 00 00 01 or 00 00 02 + outputDesc.reinitialize(buffer.data, buffer.offset, 3); + return true; + } + else if (buffer.data[buffer.offset+2] == 0x03) + { + // 00 00 03 is special because it's a subsequence of the + // NAL wrapping substitute for 00 00 00, 00 00 01, 00 00 02, + // or 00 00 03 in the RBSP sequence. We need to check the next + // byte to see whether it's 00, 01, 02, or 03 (a valid RBSP substitution) + // or whether it's something else + + if (buffer.length < 4) + return false; + + if (buffer.data[buffer.offset+3] >= 0x00 && + buffer.data[buffer.offset+3] <= 0x03) + { + // It's not really a special sequence after all + return false; + } + else + { + // It's not a standard replacement so it's a special sequence + outputDesc.reinitialize(buffer.data, buffer.offset, 3); + return true; + } + } + } + + return false; + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoPacket.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoPacket.java new file mode 100644 index 00000000..a74da573 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoPacket.java @@ -0,0 +1,56 @@ +package com.limelight.nvstream.av.video; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import com.limelight.nvstream.av.ByteBufferDescriptor; + +public class VideoPacket { + private ByteBufferDescriptor buffer; + + private int frameIndex; + private int packetIndex; + private int totalPackets; + private int payloadLength; + + public VideoPacket(ByteBufferDescriptor rtpPayload) + { + buffer = new ByteBufferDescriptor(rtpPayload); + + ByteBuffer bb = ByteBuffer.wrap(buffer.data).order(ByteOrder.LITTLE_ENDIAN); + bb.position(buffer.offset); + + frameIndex = bb.getInt(); + packetIndex = bb.getInt(); + totalPackets = bb.getInt(); + + bb.position(bb.position()+4); + + payloadLength = bb.getInt(); + } + + public int getFrameIndex() + { + return frameIndex; + } + + public int getPacketIndex() + { + return packetIndex; + } + + public int getPayloadLength() + { + return payloadLength; + } + + public int getTotalPackets() + { + return totalPackets; + } + + public ByteBufferDescriptor getNewPayloadDescriptor() + { + return new ByteBufferDescriptor(buffer.data, buffer.offset+56, buffer.length-56); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java b/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java new file mode 100644 index 00000000..0abd3bd1 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/video/VideoStream.java @@ -0,0 +1,290 @@ +package com.limelight.nvstream.av.video; + +import java.io.IOException; +import java.io.InputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketException; +import java.util.LinkedList; +import java.util.concurrent.LinkedBlockingQueue; + +import com.limelight.nvstream.NvConnectionListener; +import com.limelight.nvstream.av.ByteBufferDescriptor; +import com.limelight.nvstream.av.DecodeUnit; +import com.limelight.nvstream.av.RtpPacket; +import com.limelight.nvstream.av.ConnectionStatusListener; + +public class VideoStream { + public static final int RTP_PORT = 47998; + public static final int RTCP_PORT = 47999; + public static final int FIRST_FRAME_PORT = 47996; + + public static final int FIRST_FRAME_TIMEOUT = 5000; + + private LinkedBlockingQueue packets = new LinkedBlockingQueue(100); + + private InetAddress host; + private DatagramSocket rtp; + private Socket firstFrameSocket; + + private LinkedList threads = new LinkedList(); + + private NvConnectionListener listener; + private VideoDepacketizer depacketizer; + + private VideoDecoderRenderer decRend; + private boolean startedRendering; + + private boolean aborting = false; + + public VideoStream(InetAddress host, NvConnectionListener listener, ConnectionStatusListener avConnListener) + { + this.host = host; + this.listener = listener; + this.depacketizer = new VideoDepacketizer(avConnListener); + } + + public void abort() + { + if (aborting) { + return; + } + + aborting = true; + + // Interrupt threads + for (Thread t : threads) { + t.interrupt(); + } + + // Close the socket to interrupt the receive thread + if (rtp != null) { + rtp.close(); + } + if (firstFrameSocket != null) { + try { + firstFrameSocket.close(); + } catch (IOException e) {} + } + + // Wait for threads to terminate + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { } + } + + if (startedRendering) { + decRend.stop(); + } + + if (decRend != null) { + decRend.release(); + } + + threads.clear(); + } + + private void readFirstFrame() throws IOException + { + byte[] firstFrame = new byte[1500]; + + firstFrameSocket = new Socket(); + firstFrameSocket.setSoTimeout(FIRST_FRAME_TIMEOUT); + + try { + firstFrameSocket.connect(new InetSocketAddress(host, FIRST_FRAME_PORT), FIRST_FRAME_TIMEOUT); + InputStream firstFrameStream = firstFrameSocket.getInputStream(); + + int offset = 0; + for (;;) + { + int bytesRead = firstFrameStream.read(firstFrame, offset, firstFrame.length-offset); + + if (bytesRead == -1) + break; + + offset += bytesRead; + } + + depacketizer.addInputData(new VideoPacket(new ByteBufferDescriptor(firstFrame, 0, offset))); + } finally { + firstFrameSocket.close(); + firstFrameSocket = null; + } + } + + public void setupRtpSession() throws SocketException + { + rtp = new DatagramSocket(RTP_PORT); + } + + public void setupDecoderRenderer(VideoDecoderRenderer decRend, Object renderTarget, int drFlags) { + this.decRend = decRend; + if (decRend != null) { + decRend.setup(1280, 720, renderTarget, drFlags); + } + } + + public void startVideoStream(VideoDecoderRenderer decRend, Object renderTarget, int drFlags) throws IOException + { + // Setup the decoder and renderer + setupDecoderRenderer(decRend, renderTarget, drFlags); + + // Open RTP sockets and start session + setupRtpSession(); + + // Start pinging before reading the first frame + // so Shield Proxy knows we're here and sends us + // the reference frame + startUdpPingThread(); + + // Read the first frame to start the UDP video stream + // This MUST be called before the normal UDP receive thread + // starts in order to avoid state corruption caused by two + // threads simultaneously adding input data. + readFirstFrame(); + + if (decRend != null) { + // Start the receive thread early to avoid missing + // early packets + startReceiveThread(); + + // Start the depacketizer thread to deal with the RTP data + startDepacketizerThread(); + + // Start decoding the data we're receiving + startDecoderThread(); + + // Start the renderer + decRend.start(); + startedRendering = true; + } + } + + private void startDecoderThread() + { + Thread t = new Thread() { + @Override + public void run() { + // Read the decode units generated from the RTP stream + while (!isInterrupted()) + { + DecodeUnit du; + + try { + du = depacketizer.getNextDecodeUnit(); + } catch (InterruptedException e) { + listener.connectionTerminated(e); + return; + } + + decRend.submitDecodeUnit(du); + } + } + }; + threads.add(t); + t.setName("Video - Decoder"); + t.setPriority(Thread.MAX_PRIORITY); + t.start(); + } + + private void startDepacketizerThread() + { + // This thread lessens the work on the receive thread + // so it can spend more time waiting for data + Thread t = new Thread() { + @Override + public void run() { + while (!isInterrupted()) + { + RtpPacket packet; + + try { + packet = packets.take(); + } catch (InterruptedException e) { + listener.connectionTerminated(e); + return; + } + + // !!! We no longer own the data buffer at this point !!! + depacketizer.addInputData(packet); + } + } + }; + threads.add(t); + t.setName("Video - Depacketizer"); + t.start(); + } + + private void startReceiveThread() + { + // Receive thread + Thread t = new Thread() { + @Override + public void run() { + ByteBufferDescriptor desc = new ByteBufferDescriptor(new byte[1500], 0, 1500); + DatagramPacket packet = new DatagramPacket(desc.data, desc.length); + + while (!isInterrupted()) + { + try { + rtp.receive(packet); + } catch (IOException e) { + listener.connectionTerminated(e); + return; + } + + // Give the packet to the depacketizer thread + desc.length = packet.getLength(); + if (packets.offer(new RtpPacket(desc))) { + desc.reinitialize(new byte[1500], 0, 1500); + packet.setData(desc.data, desc.offset, desc.length); + } + } + } + }; + threads.add(t); + t.setName("Video - Receive"); + t.start(); + } + + private void startUdpPingThread() + { + // Ping thread + Thread t = new Thread() { + @Override + public void run() { + // PING in ASCII + final byte[] pingPacketData = new byte[] {0x50, 0x49, 0x4E, 0x47}; + DatagramPacket pingPacket = new DatagramPacket(pingPacketData, pingPacketData.length); + pingPacket.setSocketAddress(new InetSocketAddress(host, RTP_PORT)); + + // Send PING every 100 ms + while (!isInterrupted()) + { + try { + rtp.send(pingPacket); + } catch (IOException e) { + listener.connectionTerminated(e); + return; + } + + try { + Thread.sleep(100); + } catch (InterruptedException e) { + listener.connectionTerminated(e); + return; + } + } + } + }; + threads.add(t); + t.setName("Video - Ping"); + t.setPriority(Thread.MIN_PRIORITY); + t.start(); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/av/video/cpu/AvcDecoder.java b/moonlight-common/src/com/limelight/nvstream/av/video/cpu/AvcDecoder.java new file mode 100644 index 00000000..874a645e --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/av/video/cpu/AvcDecoder.java @@ -0,0 +1,44 @@ +package com.limelight.nvstream.av.video.cpu; + +public class AvcDecoder { + static { + // FFMPEG dependencies + System.loadLibrary("avutil-52"); + System.loadLibrary("swresample-0"); + System.loadLibrary("swscale-2"); + System.loadLibrary("avcodec-55"); + System.loadLibrary("avformat-55"); + System.loadLibrary("avfilter-3"); + + System.loadLibrary("nv_avc_dec"); + } + + /** Disables the deblocking filter at the cost of image quality */ + public static final int DISABLE_LOOP_FILTER = 0x1; + /** Uses the low latency decode flag (disables multithreading) */ + public static final int LOW_LATENCY_DECODE = 0x2; + /** Threads process each slice, rather than each frame */ + public static final int SLICE_THREADING = 0x4; + /** Uses nonstandard speedup tricks */ + public static final int FAST_DECODE = 0x8; + /** Uses bilinear filtering instead of bicubic */ + public static final int BILINEAR_FILTERING = 0x10; + /** Uses a faster bilinear filtering with lower image quality */ + public static final int FAST_BILINEAR_FILTERING = 0x20; + /** Disables color conversion (output is NV21) */ + public static final int NO_COLOR_CONVERSION = 0x40; + + public static native int init(int width, int height, int perflvl, int threadcount); + public static native void destroy(); + + // Rendering API when NO_COLOR_CONVERSION == 0 + public static native boolean setRenderTarget(Object androidSurface); + public static native boolean getRgbFrame(byte[] rgbFrame, int bufferSize); + public static native boolean redraw(); + + // Rendering API when NO_COLOR_CONVERSION == 1 + public static native boolean getRawFrame(byte[] yuvFrame, int bufferSize); + + public static native int getInputPaddingSize(); + public static native int decode(byte[] indata, int inoff, int inlen); +} diff --git a/moonlight-common/src/com/limelight/nvstream/http/NvApp.java b/moonlight-common/src/com/limelight/nvstream/http/NvApp.java new file mode 100644 index 00000000..70ef03ea --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/http/NvApp.java @@ -0,0 +1,31 @@ +package com.limelight.nvstream.http; + +public class NvApp { + private String appName; + private int appId; + private boolean isRunning; + + public void setAppName(String appName) { + this.appName = appName; + } + + public void setAppId(String appId) { + this.appId = Integer.parseInt(appId); + } + + public void setIsRunning(String isRunning) { + this.isRunning = isRunning.equals("1"); + } + + public String getAppName() { + return this.appName; + } + + public int getAppId() { + return this.appId; + } + + public boolean getIsRunning() { + return this.isRunning; + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/http/NvHTTP.java b/moonlight-common/src/com/limelight/nvstream/http/NvHTTP.java new file mode 100644 index 00000000..374bd650 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/http/NvHTTP.java @@ -0,0 +1,145 @@ +package com.limelight.nvstream.http; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.URL; +import java.net.URLConnection; +import java.util.LinkedList; +import java.util.Stack; + +import org.xmlpull.v1.XmlPullParser; +import org.xmlpull.v1.XmlPullParserException; +import org.xmlpull.v1.XmlPullParserFactory; + + +public class NvHTTP { + private String macAddress; + private String deviceName; + + public static final int PORT = 47989; + public static final int CONNECTION_TIMEOUT = 5000; + + public String baseUrl; + + public NvHTTP(InetAddress host, String macAddress, String deviceName) { + this.macAddress = macAddress; + this.deviceName = deviceName; + this.baseUrl = "http://" + host.getHostAddress() + ":" + PORT; + } + + private String getXmlString(InputStream in, String tagname) + throws XmlPullParserException, IOException { + XmlPullParserFactory factory = XmlPullParserFactory.newInstance(); + factory.setNamespaceAware(true); + XmlPullParser xpp = factory.newPullParser(); + + xpp.setInput(new InputStreamReader(in)); + int eventType = xpp.getEventType(); + Stack currentTag = new Stack(); + + while (eventType != XmlPullParser.END_DOCUMENT) { + switch (eventType) { + case (XmlPullParser.START_TAG): + currentTag.push(xpp.getName()); + break; + case (XmlPullParser.END_TAG): + currentTag.pop(); + break; + case (XmlPullParser.TEXT): + if (currentTag.peek().equals(tagname)) { + return xpp.getText(); + } + break; + } + eventType = xpp.next(); + } + + return null; + } + + private InputStream openHttpConnection(String url) throws IOException { + URLConnection conn = new URL(url).openConnection(); + conn.setConnectTimeout(CONNECTION_TIMEOUT); + conn.setDefaultUseCaches(false); + conn.connect(); + return conn.getInputStream(); + } + + public String getAppVersion() throws XmlPullParserException, IOException { + InputStream in = openHttpConnection(baseUrl + "/appversion"); + return getXmlString(in, "appversion"); + } + + public boolean getPairState() throws IOException, XmlPullParserException { + InputStream in = openHttpConnection(baseUrl + "/pairstate?mac=" + macAddress); + String paired = getXmlString(in, "paired"); + return Integer.valueOf(paired) != 0; + } + + public int getSessionId() throws IOException, XmlPullParserException { + InputStream in = openHttpConnection(baseUrl + "/pair?mac=" + macAddress + + "&devicename=" + deviceName); + String sessionId = getXmlString(in, "sessionid"); + return Integer.parseInt(sessionId); + } + + public int getSteamAppId(int sessionId) throws IOException, + XmlPullParserException { + LinkedList appList = getAppList(sessionId); + for (NvApp app : appList) { + if (app.getAppName().equals("Steam")) { + return app.getAppId(); + } + } + return 0; + } + + public LinkedList getAppList(int sessionId) throws IOException, XmlPullParserException { + InputStream in = openHttpConnection(baseUrl + "/applist?session=" + sessionId); + XmlPullParserFactory factory = XmlPullParserFactory.newInstance(); + factory.setNamespaceAware(true); + XmlPullParser xpp = factory.newPullParser(); + + xpp.setInput(new InputStreamReader(in)); + int eventType = xpp.getEventType(); + LinkedList appList = new LinkedList(); + Stack currentTag = new Stack(); + + while (eventType != XmlPullParser.END_DOCUMENT) { + switch (eventType) { + case (XmlPullParser.START_TAG): + currentTag.push(xpp.getName()); + if (xpp.getName().equals("App")) { + appList.addLast(new NvApp()); + } + break; + case (XmlPullParser.END_TAG): + currentTag.pop(); + break; + case (XmlPullParser.TEXT): + NvApp app = appList.getLast(); + if (currentTag.peek().equals("AppTitle")) { + app.setAppName(xpp.getText()); + } else if (currentTag.peek().equals("ID")) { + app.setAppId(xpp.getText()); + } else if (currentTag.peek().equals("IsRunning")) { + app.setIsRunning(xpp.getText()); + } + break; + } + eventType = xpp.next(); + } + return appList; + } + + // Returns gameSession XML attribute + public int launchApp(int sessionId, int appId) throws IOException, + XmlPullParserException { + InputStream in = openHttpConnection(baseUrl + "/launch?session=" + + sessionId + "&appid=" + appId); + String gameSession = getXmlString(in, "gamesession"); + return Integer.parseInt(gameSession); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/input/ControllerPacket.java b/moonlight-common/src/com/limelight/nvstream/input/ControllerPacket.java new file mode 100644 index 00000000..b36810bf --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/input/ControllerPacket.java @@ -0,0 +1,89 @@ +package com.limelight.nvstream.input; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class ControllerPacket extends InputPacket { + public static final byte[] HEADER = + { + 0x0A, + 0x00, + 0x00, + 0x00, + 0x00, + 0x14 + }; + + public static final byte[] TAIL = + { + (byte)0x9C, + 0x00, + 0x00, + 0x00, + 0x55, + 0x00 + }; + + public static final int PACKET_TYPE = 0x18; + + public static final short A_FLAG = 0x1000; + public static final short B_FLAG = 0x2000; + public static final short X_FLAG = 0x4000; + public static final short Y_FLAG = (short)0x8000; + public static final short UP_FLAG = 0x0001; + public static final short DOWN_FLAG = 0x0002; + public static final short LEFT_FLAG = 0x0004; + public static final short RIGHT_FLAG = 0x0008; + public static final short LB_FLAG = 0x0100; + public static final short RB_FLAG = 0x0200; + public static final short PLAY_FLAG = 0x0010; + public static final short BACK_FLAG = 0x0020; + public static final short LS_CLK_FLAG = 0x0040; + public static final short RS_CLK_FLAG = 0x0080; + public static final short SPECIAL_BUTTON_FLAG = 0x0400; + + public static final short PAYLOAD_LENGTH = 24; + public static final short PACKET_LENGTH = PAYLOAD_LENGTH + + InputPacket.HEADER_LENGTH; + + private short buttonFlags; + private byte leftTrigger; + private byte rightTrigger; + private short leftStickX; + private short leftStickY; + private short rightStickX; + private short rightStickY; + + public ControllerPacket(short buttonFlags, byte leftTrigger, byte rightTrigger, + short leftStickX, short leftStickY, + short rightStickX, short rightStickY) + { + super(PACKET_TYPE); + + this.buttonFlags = buttonFlags; + this.leftTrigger = leftTrigger; + this.rightTrigger = rightTrigger; + this.leftStickX = leftStickX; + this.leftStickY = leftStickY; + this.rightStickX = rightStickX; + this.rightStickY = rightStickY; + } + + public byte[] toWire() + { + ByteBuffer bb = ByteBuffer.allocate(PACKET_LENGTH).order(ByteOrder.LITTLE_ENDIAN); + + bb.put(toWireHeader()); + bb.put(HEADER); + bb.putShort(buttonFlags); + bb.put(leftTrigger); + bb.put(rightTrigger); + bb.putShort(leftStickX); + bb.putShort(leftStickY); + bb.putShort(rightStickX); + bb.putShort(rightStickY); + bb.put(TAIL); + + return bb.array(); + } + } \ No newline at end of file diff --git a/moonlight-common/src/com/limelight/nvstream/input/InputPacket.java b/moonlight-common/src/com/limelight/nvstream/input/InputPacket.java new file mode 100644 index 00000000..dec3ff11 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/input/InputPacket.java @@ -0,0 +1,26 @@ +package com.limelight.nvstream.input; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public abstract class InputPacket { + public static final int HEADER_LENGTH = 0x4; + + protected int packetType; + + public InputPacket(int packetType) + { + this.packetType = packetType; + } + + public abstract byte[] toWire(); + + public byte[] toWireHeader() + { + ByteBuffer bb = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN); + + bb.putInt(packetType); + + return bb.array(); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/input/MouseButtonPacket.java b/moonlight-common/src/com/limelight/nvstream/input/MouseButtonPacket.java new file mode 100644 index 00000000..70cea01c --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/input/MouseButtonPacket.java @@ -0,0 +1,36 @@ +package com.limelight.nvstream.input; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class MouseButtonPacket extends InputPacket { + + private byte buttonEventType; + + public static final int PACKET_TYPE = 0x5; + public static final int PAYLOAD_LENGTH = 5; + public static final int PACKET_LENGTH = PAYLOAD_LENGTH + + InputPacket.HEADER_LENGTH; + + public static final byte PRESS_EVENT = 0x07; + public static final byte RELEASE_EVENT = 0x08; + + public MouseButtonPacket(boolean leftButtonDown) + { + super(PACKET_TYPE); + + buttonEventType = leftButtonDown ? + PRESS_EVENT : RELEASE_EVENT; + } + + @Override + public byte[] toWire() { + ByteBuffer bb = ByteBuffer.allocate(PACKET_LENGTH).order(ByteOrder.BIG_ENDIAN); + + bb.put(toWireHeader()); + bb.put(buttonEventType); + bb.putInt(1); // FIXME: button index? + + return bb.array(); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/input/MouseMovePacket.java b/moonlight-common/src/com/limelight/nvstream/input/MouseMovePacket.java new file mode 100644 index 00000000..dba6f27b --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/input/MouseMovePacket.java @@ -0,0 +1,42 @@ +package com.limelight.nvstream.input; + +import java.nio.ByteBuffer; + +public class MouseMovePacket extends InputPacket { + + private static final byte[] HEADER = + { + 0x06, + 0x00, + 0x00, + 0x00 + }; + + public static final int PACKET_TYPE = 0x8; + public static final int PAYLOAD_LENGTH = 8; + public static final int PACKET_LENGTH = PAYLOAD_LENGTH + + InputPacket.HEADER_LENGTH; + + private short deltaX; + private short deltaY; + + public MouseMovePacket(short deltaX, short deltaY) + { + super(PACKET_TYPE); + + this.deltaX = deltaX; + this.deltaY = deltaY; + } + + @Override + public byte[] toWire() { + ByteBuffer bb = ByteBuffer.allocate(PACKET_LENGTH); + + bb.put(toWireHeader()); + bb.put(HEADER); + bb.putShort(deltaX); + bb.putShort(deltaY); + + return bb.array(); + } +} diff --git a/moonlight-common/src/com/limelight/nvstream/input/NvController.java b/moonlight-common/src/com/limelight/nvstream/input/NvController.java new file mode 100644 index 00000000..b0b85984 --- /dev/null +++ b/moonlight-common/src/com/limelight/nvstream/input/NvController.java @@ -0,0 +1,65 @@ +package com.limelight.nvstream.input; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; + +public class NvController { + + public final static int PORT = 35043; + + public final static int CONTROLLER_TIMEOUT = 3000; + + private InetAddress host; + private Socket s; + private OutputStream out; + + public NvController(InetAddress host) + { + this.host = host; + } + + public void initialize() throws IOException + { + s = new Socket(); + s.connect(new InetSocketAddress(host, PORT), CONTROLLER_TIMEOUT); + s.setTcpNoDelay(true); + out = s.getOutputStream(); + } + + public void close() + { + try { + s.close(); + } catch (IOException e) {} + } + + public void sendControllerInput(short buttonFlags, byte leftTrigger, byte rightTrigger, + short leftStickX, short leftStickY, short rightStickX, short rightStickY) throws IOException + { + out.write(new ControllerPacket(buttonFlags, leftTrigger, + rightTrigger, leftStickX, leftStickY, + rightStickX, rightStickY).toWire()); + out.flush(); + } + + public void sendMouseButtonDown() throws IOException + { + out.write(new MouseButtonPacket(true).toWire()); + out.flush(); + } + + public void sendMouseButtonUp() throws IOException + { + out.write(new MouseButtonPacket(false).toWire()); + out.flush(); + } + + public void sendMouseMove(short deltaX, short deltaY) throws IOException + { + out.write(new MouseMovePacket(deltaX, deltaY).toWire()); + out.flush(); + } +}