moonlight-stream/moonlight-android
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-android.git synced 2026-02-16 10:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial implementation of AV1

Browse Source
This commit is contained in:
Cameron Gutman
2023-07-02 22:49:42 -05:00
parent d6bbfa1af1
commit 3a78095574
8 changed files with 124 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
app/src/main/java/com/limelight/Game.java
View File

@@ -405,9 +405,9 @@ public class Game extends Activity implements SurfaceHolder.Callback,
this);
// Don't stream HDR if the decoder can't support it
if (willStreamHdr && !decoderRenderer.isHevcMain10Hdr10Supported()) {
if (willStreamHdr && !decoderRenderer.isHevcMain10Hdr10Supported() && !decoderRenderer.isAv1Main10Supported()) {
willStreamHdr = false;
Toast.makeText(this, "Decoder does not support HEVC Main10HDR10", Toast.LENGTH_LONG).show();
Toast.makeText(this, "Decoder does not support HDR10 profile", Toast.LENGTH_LONG).show();
}
// Display a message to the user if HEVC was forced on but we still didn't find a decoder
@@ -415,6 +415,23 @@ public class Game extends Activity implements SurfaceHolder.Callback,
Toast.makeText(this, "No HEVC decoder found.\nFalling back to H.264.", Toast.LENGTH_LONG).show();
}
// TODO: Display a message to the user if HEVC was forced on but we still didn't find a decoder
// H.264 is always supported
int supportedVideoFormats = MoonBridge.VIDEO_FORMAT_H264;
if (decoderRenderer.isHevcSupported()) {
supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_H265;
if (decoderRenderer.isHevcMain10Hdr10Supported()) {
supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_H265_MAIN10;
}
}
if (decoderRenderer.isAv1Supported()) {
supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_AV1_MAIN8;
if (decoderRenderer.isAv1Main10Supported()) {
supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_AV1_MAIN10;
}
}
int gamepadMask = ControllerHandler.getAttachedControllerMask(this);
if (!prefConfig.multiController) {
// Always set gamepad 1 present for when multi-controller is
@@ -464,7 +481,8 @@ public class Game extends Activity implements SurfaceHolder.Callback,
.setMaxPacketSize(1392)
.setRemoteConfiguration(StreamConfiguration.STREAM_CFG_AUTO) // NvConnection will perform LAN and VPN detection
.setHevcBitratePercentageMultiplier(75)
.setHevcSupported(decoderRenderer.isHevcSupported())
.setAv1BitratePercentageMultiplier(60)
.setSupportedVideoFormats(supportedVideoFormats)
.setEnableHdr(willStreamHdr)
.setAttachedGamepadMask(gamepadMask)
.setClientRefreshRateX100((int)(displayRefreshRate * 100))

60
app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java
View File

@@ -46,6 +46,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
private MediaCodecInfo avcDecoder;
private MediaCodecInfo hevcDecoder;
private MediaCodecInfo av1Decoder;
private byte[] vpsBuffer;
private byte[] spsBuffer;
@@ -64,7 +65,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
private boolean needsSpsBitstreamFixup, isExynos4;
private boolean adaptivePlayback, directSubmit, fusedIdrFrame;
private boolean constrainedHighProfile;
private boolean refFrameInvalidationAvc, refFrameInvalidationHevc;
private boolean refFrameInvalidationAvc, refFrameInvalidationHevc, refFrameInvalidationAv1;
private byte optimalSlicesPerFrame;
private boolean refFrameInvalidationActive;
private int initialWidth, initialHeight;
@@ -231,6 +232,10 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
return hevcDecoderInfo;
}
private MediaCodecInfo findAv1Decoder(PreferenceConfiguration prefs) {
return MediaCodecHelper.findProbableSafeDecoder("video/av01", -1);
}
public void setRenderTarget(SurfaceHolder renderTarget) {
this.renderTarget = renderTarget;
}
@@ -269,6 +274,14 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
LimeLog.info("No HEVC decoder found");
}
av1Decoder = findAv1Decoder(prefs);
if (av1Decoder != null) {
LimeLog.info("Selected AV1 decoder: "+av1Decoder.getName());
}
else {
LimeLog.info("No AV1 decoder found");
}
// Set attributes that are queried in getCapabilities(). This must be done here
// because getCapabilities() may be called before setup() in current versions of the common
// library. The limitation of this is that we don't know whether we're using HEVC or AVC.
@@ -299,6 +312,14 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
LimeLog.info("Decoder "+hevcDecoder.getName()+" wants "+hevcOptimalSlicesPerFrame+" slices per frame");
}
if (av1Decoder != null) {
refFrameInvalidationAv1 = MediaCodecHelper.decoderSupportsRefFrameInvalidationAv1(av1Decoder);
if (refFrameInvalidationAv1) {
LimeLog.info("Decoder "+av1Decoder.getName()+" will use reference frame invalidation for AV1");
}
}
// Use the larger of the two slices per frame preferences
optimalSlicesPerFrame = (byte)Math.max(avcOptimalSlicesPerFrame, hevcOptimalSlicesPerFrame);
LimeLog.info("Requesting "+optimalSlicesPerFrame+" slices per frame");
@@ -332,6 +353,25 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
return false;
}
public boolean isAv1Supported() {
return av1Decoder != null;
}
public boolean isAv1Main10Supported() {
if (av1Decoder == null) {
return false;
}
for (MediaCodecInfo.CodecProfileLevel profileLevel : av1Decoder.getCapabilitiesForType("video/av01").profileLevels) {
if (profileLevel.profile == MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10) {
LimeLog.info("AV1 decoder "+av1Decoder.getName()+" supports AV1 Main 10 HDR10");
return true;
}
}
return false;
}
public int getPreferredColorSpace() {
// Default to Rec 709 which is probably better supported on modern devices.
//
@@ -340,7 +380,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
// an HEVC decoder, we will use Rec 709 (even for H.264) since we can't choose a
// colorspace by codec (and it's probably safe to say a SoC with HEVC decoding is
// plenty modern enough to handle H.264 VUI colorspace info).
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O || hevcDecoder != null) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O || hevcDecoder != null || av1Decoder != null) {
return MoonBridge.COLORSPACE_REC_709;
}
else {
@@ -551,6 +591,17 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
refFrameInvalidationActive = refFrameInvalidationHevc;
}
else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_AV1) != 0) {
mimeType = "video/av01";
selectedDecoderInfo = av1Decoder;
if (av1Decoder == null) {
LimeLog.severe("No available AV1 decoder!");
return -2;
}
refFrameInvalidationActive = refFrameInvalidationAv1;
}
else {
// Unknown format
LimeLog.severe("Unknown format");
@@ -1324,6 +1375,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
decoder = avcDecoder.getName();
} else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H265) != 0) {
decoder = hevcDecoder.getName();
} else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_AV1) != 0) {
decoder = av1Decoder.getName();
} else {
decoder = "(unknown)";
}
@@ -1670,6 +1723,9 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C
if (refFrameInvalidationHevc) {
capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC;
}
if (refFrameInvalidationAv1) {
capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_AV1;
}
// Enable direct submit on supported hardware
if (directSubmit) {

11
app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java
View File

@@ -691,6 +691,17 @@ public class MediaCodecHelper {
return isDecoderInList(refFrameInvalidationHevcPrefixes, decoderInfo.getName());
}
public static boolean decoderSupportsRefFrameInvalidationAv1(MediaCodecInfo decoderInfo) {
// We'll use the same heuristics as HEVC for now
if (decoderSupportsAndroidRLowLatency(decoderInfo, "video/av01") ||
decoderSupportsKnownVendorLowLatencyOption(decoderInfo.getName())) {
LimeLog.info("Enabling AV1 RFI based on low latency option support");
return true;
}
return false;
}
public static boolean decoderIsWhitelistedForHevc(MediaCodecInfo decoderInfo) {
// Google didn't have official support for HEVC (or more importantly, a CTS test) until
// Lollipop. I've seen some MediaTek devices on 4.4 crash when attempting to use HEVC,

7
app/src/main/java/com/limelight/nvstream/NvConnection.java
View File

@@ -259,8 +259,8 @@ public class NvConnection {
return false;
}
else if ((context.streamConfig.getWidth() > 4096 || context.streamConfig.getHeight() > 4096) &&
!context.streamConfig.getHevcSupported()) {
context.connListener.displayMessage("Your streaming device must support HEVC to stream at resolutions above 4K.");
(context.streamConfig.getSupportedVideoFormats() & ~MoonBridge.VIDEO_FORMAT_MASK_H264) == 0) {
context.connListener.displayMessage("Your streaming device must support HEVC or AV1 to stream at resolutions above 4K.");
return false;
}
else if (context.streamConfig.getHeight() >= 2160 && !h.supports4K(serverInfo)) {
@@ -429,9 +429,10 @@ public class NvConnection {
context.streamConfig.getRefreshRate(), context.streamConfig.getBitrate(),
context.negotiatedPacketSize, context.negotiatedRemoteStreaming,
context.streamConfig.getAudioConfiguration().toInt(),
context.streamConfig.getHevcSupported(),
context.streamConfig.getSupportedVideoFormats(),
context.negotiatedHdr,
context.streamConfig.getHevcBitratePercentageMultiplier(),
context.streamConfig.getAv1BitratePercentageMultiplier(),
context.streamConfig.getClientRefreshRateX100(),
context.streamConfig.getEncryptionFlags(),
context.riKey.getEncoded(), ib.array(),

23
app/src/main/java/com/limelight/nvstream/StreamConfiguration.java
View File

@@ -22,8 +22,9 @@ public class StreamConfiguration {
private int maxPacketSize;
private int remote;
private MoonBridge.AudioConfiguration audioConfiguration;
private boolean supportsHevc;
private int supportedVideoFormats;
private int hevcBitratePercentageMultiplier;
private int av1BitratePercentageMultiplier;
private boolean enableHdr;
private int attachedGamepadMask;
private int encryptionFlags;
@@ -90,6 +91,11 @@ public class StreamConfiguration {
return this;
}
public StreamConfiguration.Builder setAv1BitratePercentageMultiplier(int multiplier) {
config.av1BitratePercentageMultiplier = multiplier;
return this;
}
public StreamConfiguration.Builder setEnableHdr(boolean enableHdr) {
config.enableHdr = enableHdr;
return this;
@@ -135,8 +141,8 @@ public class StreamConfiguration {
return this;
}
public StreamConfiguration.Builder setHevcSupported(boolean supportsHevc) {
config.supportsHevc = supportsHevc;
public StreamConfiguration.Builder setSupportedVideoFormats(int supportedVideoFormats) {
config.supportedVideoFormats = supportedVideoFormats;
return this;
}
@@ -168,8 +174,7 @@ public class StreamConfiguration {
this.sops = true;
this.enableAdaptiveResolution = false;
this.audioConfiguration = MoonBridge.AUDIO_CONFIGURATION_STEREO;
this.supportsHevc = false;
this.enableHdr = false;
this.supportedVideoFormats = MoonBridge.VIDEO_FORMAT_H264;
this.attachedGamepadMask = 0;
}
@@ -221,14 +226,18 @@ public class StreamConfiguration {
return audioConfiguration;
}
public boolean getHevcSupported() {
return supportsHevc;
public int getSupportedVideoFormats() {
return supportedVideoFormats;
}
public int getHevcBitratePercentageMultiplier() {
return hevcBitratePercentageMultiplier;
}
public int getAv1BitratePercentageMultiplier() {
return av1BitratePercentageMultiplier;
}
public boolean getEnableHdr() {
return enableHdr;
}

12
app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java
View File

@@ -14,9 +14,13 @@ public class MoonBridge {
public static final int VIDEO_FORMAT_H264 = 0x0001;
public static final int VIDEO_FORMAT_H265 = 0x0100;
public static final int VIDEO_FORMAT_H265_MAIN10 = 0x0200;
public static final int VIDEO_FORMAT_AV1_MAIN8 = 0x1000;
public static final int VIDEO_FORMAT_AV1_MAIN10 = 0x2000;
public static final int VIDEO_FORMAT_MASK_H264 = 0x00FF;
public static final int VIDEO_FORMAT_MASK_H265 = 0xFF00;
public static final int VIDEO_FORMAT_MASK_H264 = 0x000F;
public static final int VIDEO_FORMAT_MASK_H265 = 0x0F00;
public static final int VIDEO_FORMAT_MASK_AV1 = 0xF000;
public static final int VIDEO_FORMAT_MASK_10BIT = 0x2200;
public static final int ENCFLG_NONE = 0;
public static final int ENCFLG_AUDIO = 1;
@@ -40,6 +44,7 @@ public class MoonBridge {
public static final int CAPABILITY_DIRECT_SUBMIT = 1;
public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC = 2;
public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC = 4;
public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_AV1 = 0x40;
public static final int DR_OK = 0;
public static final int DR_NEED_IDR = -1;
@@ -340,9 +345,10 @@ public class MoonBridge {
String rtspSessionUrl,
int width, int height, int fps,
int bitrate, int packetSize, int streamingRemotely,
int audioConfiguration, boolean supportsHevc,
int audioConfiguration, int supportedVideoFormats,
boolean enableHdr,
int hevcBitratePercentageMultiplier,
int av1BitratePercentageMultiplier,
int clientRefreshRateX100,
int encryptionFlags,
byte[] riAesKey, byte[] riAesIv,

6
app/src/main/jni/moonlight-core/callbacks.c
View File

@@ -432,9 +432,10 @@ Java_com_limelight_nvstream_jni_MoonBridge_startConnection(JNIEnv *env, jclass c
jstring rtspSessionUrl,
jint width, jint height, jint fps,
jint bitrate, jint packetSize, jint streamingRemotely,
jint audioConfiguration, jboolean supportsHevc,
jint audioConfiguration, jint supportedVideoFormats,
jboolean enableHdr,
jint hevcBitratePercentageMultiplier,
jint av1BitratePercentageMultiplier,
jint clientRefreshRateX100,
jint encryptionFlags,
jbyteArray riAesKey, jbyteArray riAesIv,
@@ -454,9 +455,10 @@ Java_com_limelight_nvstream_jni_MoonBridge_startConnection(JNIEnv *env, jclass c
.packetSize = packetSize,
.streamingRemotely = streamingRemotely,
.audioConfiguration = audioConfiguration,
.supportsHevc = supportsHevc,
.supportedVideoFormats = supportedVideoFormats,
.enableHdr = enableHdr,
.hevcBitratePercentageMultiplier = hevcBitratePercentageMultiplier,
.av1BitratePercentageMultiplier = av1BitratePercentageMultiplier,
.clientRefreshRateX100 = clientRefreshRateX100,
.encryptionFlags = encryptionFlags,
.colorSpace = colorSpace,

2
app/src/main/jni/moonlight-core/moonlight-common-c

Submodule app/src/main/jni/moonlight-core/moonlight-common-c updated: c0792168f5...e36bde4acc