From 7951c0fa134e3ad06bf928f2cbe2aa4e84e3cb0f Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Thu, 21 Nov 2013 12:24:45 -0500 Subject: [PATCH] WIP optimization of object allocation and pools. There's currently a double free on the AvByteBufferDescriptor that I'm trying to track down. --- src/com/limelight/nvstream/NvAudioStream.java | 5 +- src/com/limelight/nvstream/NvVideoStream.java | 8 ++- .../nvstream/av/AvByteBufferDescriptor.java | 43 +++++++----- .../nvstream/av/AvByteBufferPool.java | 25 ++++--- .../limelight/nvstream/av/AvDecodeUnit.java | 19 +++++- .../limelight/nvstream/av/AvObjectPool.java | 22 +++++++ .../limelight/nvstream/av/AvRtpPacket.java | 29 +++++++- .../nvstream/av/AvShortBufferDescriptor.java | 26 +++++++- .../nvstream/av/AvShortBufferPool.java | 25 ++++--- .../av/audio/AvAudioDepacketizer.java | 8 ++- .../av/video/AvVideoDepacketizer.java | 66 +++++++++++-------- .../nvstream/av/video/AvVideoPacket.java | 27 +++++++- 12 files changed, 213 insertions(+), 90 deletions(-) create mode 100644 src/com/limelight/nvstream/av/AvObjectPool.java diff --git a/src/com/limelight/nvstream/NvAudioStream.java b/src/com/limelight/nvstream/NvAudioStream.java index e688b50c..e49202a8 100644 --- a/src/com/limelight/nvstream/NvAudioStream.java +++ b/src/com/limelight/nvstream/NvAudioStream.java @@ -168,6 +168,7 @@ public class NvAudioStream { depacketizer.decodeInputData(packet); pool.free(packet.getBackingBuffer()); + packet.free(); } } }; @@ -209,7 +210,7 @@ public class NvAudioStream { @Override public void run() { DatagramPacket packet = new DatagramPacket(pool.allocate(), 1500); - AvByteBufferDescriptor desc = new AvByteBufferDescriptor(null, 0, 0); + AvByteBufferDescriptor desc = AvByteBufferDescriptor.newDescriptor(null, 0, 0); while (!isInterrupted()) { @@ -225,7 +226,7 @@ public class NvAudioStream { desc.data = packet.getData(); // Give the packet to the depacketizer thread - packets.add(new AvRtpPacket(desc)); + packets.add(AvRtpPacket.create(desc)); // Get a new buffer from the buffer pool packet.setData(pool.allocate(), 0, 1500); diff --git a/src/com/limelight/nvstream/NvVideoStream.java b/src/com/limelight/nvstream/NvVideoStream.java index e3d903d9..44ac2867 100644 --- a/src/com/limelight/nvstream/NvVideoStream.java +++ b/src/com/limelight/nvstream/NvVideoStream.java @@ -110,7 +110,7 @@ public class NvVideoStream { } System.out.println("VID: First frame read ("+offset+" bytes)"); - depacketizer.addInputData(new AvVideoPacket(new AvByteBufferDescriptor(firstFrame, 0, offset))); + depacketizer.addInputData(AvVideoPacket.createNoCopy(AvByteBufferDescriptor.newDescriptor(firstFrame, 0, offset))); } finally { firstFrameSocket.close(); firstFrameSocket = null; @@ -249,6 +249,8 @@ public class NvVideoStream { // !!! We no longer own the data buffer at this point !!! depacketizer.addInputData(packet); + + packet.free(); } } }; @@ -263,7 +265,7 @@ public class NvVideoStream { @Override public void run() { DatagramPacket packet = new DatagramPacket(depacketizer.allocatePacketBuffer(), 1500); - AvByteBufferDescriptor desc = new AvByteBufferDescriptor(null, 0, 0); + AvByteBufferDescriptor desc = AvByteBufferDescriptor.newDescriptor(null, 0, 0); while (!isInterrupted()) { @@ -279,7 +281,7 @@ public class NvVideoStream { desc.data = packet.getData(); // Give the packet to the depacketizer thread - packets.add(new AvRtpPacket(desc)); + packets.add(AvRtpPacket.create(desc)); // Get a new buffer from the buffer pool packet.setData(depacketizer.allocatePacketBuffer(), 0, 1500); diff --git a/src/com/limelight/nvstream/av/AvByteBufferDescriptor.java b/src/com/limelight/nvstream/av/AvByteBufferDescriptor.java index b75a78ef..7e70c93a 100644 --- a/src/com/limelight/nvstream/av/AvByteBufferDescriptor.java +++ b/src/com/limelight/nvstream/av/AvByteBufferDescriptor.java @@ -6,35 +6,42 @@ public class AvByteBufferDescriptor { public int length; public Object context; - public AvByteBufferDescriptor(byte[] data, int offset, int length) + private static AvObjectPool pool = new AvObjectPool(); + public static AvByteBufferDescriptor newDescriptor(byte[] data, int offset, int length) { + AvByteBufferDescriptor buffer = pool.tryAllocate(); + if (buffer != null) { + buffer.data = data; + buffer.offset = offset; + buffer.length = length; + buffer.context = null; + return buffer; + } + else { + return new AvByteBufferDescriptor(data, offset, length); + } + } + + public static AvByteBufferDescriptor newDescriptor(AvByteBufferDescriptor buffer) { + return newDescriptor(buffer.data, buffer.offset, buffer.length); + } + + private AvByteBufferDescriptor(byte[] data, int offset, int length) { this.data = data; this.offset = offset; this.length = length; + this.context = null; } - public AvByteBufferDescriptor(AvByteBufferDescriptor desc) + private AvByteBufferDescriptor(AvByteBufferDescriptor desc) { this.data = desc.data; this.offset = desc.offset; this.length = desc.length; + this.context = null; } - public void print() - { - print(offset, length); - } - - public void print(int length) - { - print(this.offset, length); - } - - public void print(int offset, int length) - { - for (int i = offset; i < offset+length; i++) { - System.out.printf("%d: %02x \n", i, data[i]); - } - System.out.println(); + public void free() { + pool.free(this); } } diff --git a/src/com/limelight/nvstream/av/AvByteBufferPool.java b/src/com/limelight/nvstream/av/AvByteBufferPool.java index a4892eaf..1da5e6ad 100644 --- a/src/com/limelight/nvstream/av/AvByteBufferPool.java +++ b/src/com/limelight/nvstream/av/AvByteBufferPool.java @@ -1,9 +1,9 @@ package com.limelight.nvstream.av; -import java.util.LinkedList; +import java.util.concurrent.ConcurrentLinkedQueue; public class AvByteBufferPool { - private LinkedList bufferList = new LinkedList(); + private ConcurrentLinkedQueue bufferList = new ConcurrentLinkedQueue(); private int bufferSize; public AvByteBufferPool(int size) @@ -11,25 +11,22 @@ public class AvByteBufferPool { this.bufferSize = size; } - public synchronized void purge() + public void purge() { - this.bufferList = new LinkedList(); + bufferList.clear(); } - public synchronized byte[] allocate() + public byte[] allocate() { - if (bufferList.isEmpty()) - { - return new byte[bufferSize]; - } - else - { - return bufferList.removeFirst(); + byte[] buff = bufferList.poll(); + if (buff == null) { + buff = new byte[bufferSize]; } + return buff; } - public synchronized void free(byte[] buffer) + public void free(byte[] buffer) { - bufferList.addFirst(buffer); + bufferList.add(buffer); } } diff --git a/src/com/limelight/nvstream/av/AvDecodeUnit.java b/src/com/limelight/nvstream/av/AvDecodeUnit.java index 69300b83..1f4fead9 100644 --- a/src/com/limelight/nvstream/av/AvDecodeUnit.java +++ b/src/com/limelight/nvstream/av/AvDecodeUnit.java @@ -12,7 +12,19 @@ public class AvDecodeUnit { private int dataLength; private int flags; - public AvDecodeUnit(int type, List bufferList, int dataLength, int flags) + private static AvObjectPool pool = new AvObjectPool(); + public static AvDecodeUnit newDecodeUnit(int type, List bufferList, int dataLength, int flags) { + AvDecodeUnit du = pool.tryAllocate(); + if (du == null) { + du = new AvDecodeUnit(); + } + du.initialize(type, bufferList, dataLength, flags); + return du; + } + + private AvDecodeUnit() { } + + public void initialize(int type, List bufferList, int dataLength, int flags) { this.type = type; this.bufferList = bufferList; @@ -39,4 +51,9 @@ public class AvDecodeUnit { { return dataLength; } + + public void free() + { + pool.free(this); + } } diff --git a/src/com/limelight/nvstream/av/AvObjectPool.java b/src/com/limelight/nvstream/av/AvObjectPool.java new file mode 100644 index 00000000..97de612c --- /dev/null +++ b/src/com/limelight/nvstream/av/AvObjectPool.java @@ -0,0 +1,22 @@ +package com.limelight.nvstream.av; + +import java.util.concurrent.ConcurrentLinkedQueue; + +public class AvObjectPool { + private ConcurrentLinkedQueue objectList = new ConcurrentLinkedQueue(); + + public void purge() + { + objectList.clear(); + } + + public T tryAllocate() + { + return objectList.poll(); + } + + public void free(T object) + { + objectList.add(object); + } +} diff --git a/src/com/limelight/nvstream/av/AvRtpPacket.java b/src/com/limelight/nvstream/av/AvRtpPacket.java index 8e4250e7..50facdb8 100644 --- a/src/com/limelight/nvstream/av/AvRtpPacket.java +++ b/src/com/limelight/nvstream/av/AvRtpPacket.java @@ -8,9 +8,26 @@ public class AvRtpPacket { private short seqNum; private AvByteBufferDescriptor buffer; - public AvRtpPacket(AvByteBufferDescriptor buffer) + private static AvObjectPool pool = new AvObjectPool(); + + public static AvRtpPacket create(AvByteBufferDescriptor payload) { + return createNoCopy(AvByteBufferDescriptor.newDescriptor(payload)); + } + + public static AvRtpPacket createNoCopy(AvByteBufferDescriptor payload) { + AvRtpPacket pkt = pool.tryAllocate(); + if (pkt == null) { + pkt = new AvRtpPacket(); + } + pkt.initialize(payload); + return pkt; + } + + private AvRtpPacket() { } + + private void initialize(AvByteBufferDescriptor buffer) { - this.buffer = new AvByteBufferDescriptor(buffer); + this.buffer = buffer; ByteBuffer bb = ByteBuffer.wrap(buffer.data, buffer.offset, buffer.length); @@ -39,8 +56,14 @@ public class AvRtpPacket { return buffer.data; } + public void free() + { + buffer.free(); + pool.free(this); + } + public AvByteBufferDescriptor getNewPayloadDescriptor() { - return new AvByteBufferDescriptor(buffer.data, buffer.offset+12, buffer.length-12); + return AvByteBufferDescriptor.newDescriptor(buffer.data, buffer.offset+12, buffer.length-12); } } diff --git a/src/com/limelight/nvstream/av/AvShortBufferDescriptor.java b/src/com/limelight/nvstream/av/AvShortBufferDescriptor.java index 9d028326..706b5fef 100644 --- a/src/com/limelight/nvstream/av/AvShortBufferDescriptor.java +++ b/src/com/limelight/nvstream/av/AvShortBufferDescriptor.java @@ -5,17 +5,39 @@ public class AvShortBufferDescriptor { public int offset; public int length; - public AvShortBufferDescriptor(short[] data, int offset, int length) + private static AvObjectPool pool = new AvObjectPool(); + public static AvShortBufferDescriptor newDescriptor(short[] data, int offset, int length) { + AvShortBufferDescriptor buffer = pool.tryAllocate(); + if (buffer != null) { + buffer.data = data; + buffer.offset = offset; + buffer.length = length; + return buffer; + } + else { + return new AvShortBufferDescriptor(data, offset, length); + } + } + + public static AvShortBufferDescriptor newDescriptor(AvShortBufferDescriptor buffer) { + return newDescriptor(buffer.data, buffer.offset, buffer.length); + } + + private AvShortBufferDescriptor(short[] data, int offset, int length) { this.data = data; this.offset = offset; this.length = length; } - public AvShortBufferDescriptor(AvShortBufferDescriptor desc) + private AvShortBufferDescriptor(AvShortBufferDescriptor desc) { this.data = desc.data; this.offset = desc.offset; this.length = desc.length; } + + public void free() { + pool.free(this); + } } diff --git a/src/com/limelight/nvstream/av/AvShortBufferPool.java b/src/com/limelight/nvstream/av/AvShortBufferPool.java index 0b000483..012383f8 100644 --- a/src/com/limelight/nvstream/av/AvShortBufferPool.java +++ b/src/com/limelight/nvstream/av/AvShortBufferPool.java @@ -1,9 +1,9 @@ package com.limelight.nvstream.av; -import java.util.LinkedList; +import java.util.concurrent.ConcurrentLinkedQueue; public class AvShortBufferPool { - private LinkedList bufferList = new LinkedList(); + private ConcurrentLinkedQueue bufferList = new ConcurrentLinkedQueue(); private int bufferSize; public AvShortBufferPool(int size) @@ -11,25 +11,22 @@ public class AvShortBufferPool { this.bufferSize = size; } - public synchronized void purge() + public void purge() { - this.bufferList = new LinkedList(); + bufferList.clear(); } - public synchronized short[] allocate() + public short[] allocate() { - if (bufferList.isEmpty()) - { - return new short[bufferSize]; - } - else - { - return bufferList.removeFirst(); + short[] buff = bufferList.poll(); + if (buff == null) { + buff = new short[bufferSize]; } + return buff; } - public synchronized void free(short[] buffer) + public void free(short[] buffer) { - bufferList.addFirst(buffer); + bufferList.add(buffer); } } diff --git a/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java b/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java index 13a78f4b..32dca37c 100644 --- a/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java +++ b/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java @@ -32,13 +32,13 @@ public class AvAudioDepacketizer { decodeLen *= OpusDecoder.getChannelCount(); // Put it on the decoded queue - if (!decodedUnits.offer(new AvShortBufferDescriptor(pcmData, 0, decodeLen))) + AvShortBufferDescriptor sbd = AvShortBufferDescriptor.newDescriptor(pcmData, 0, decodeLen); + if (!decodedUnits.offer(sbd)) { - pool.free(pcmData); + releaseBuffer(sbd); } } else { - System.out.println("decode failed: "+decodeLen); pool.free(pcmData); } } @@ -66,11 +66,13 @@ public class AvAudioDepacketizer { // This is all the depacketizing we need to do AvByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); decodeData(rtpPayload.data, rtpPayload.offset, rtpPayload.length); + rtpPayload.free(); } public void releaseBuffer(AvShortBufferDescriptor decodedData) { pool.free(decodedData.data); + decodedData.free(); } public AvShortBufferDescriptor getNextDecodedData() throws InterruptedException diff --git a/src/com/limelight/nvstream/av/video/AvVideoDepacketizer.java b/src/com/limelight/nvstream/av/video/AvVideoDepacketizer.java index db1d6b01..646b0b7d 100644 --- a/src/com/limelight/nvstream/av/video/AvVideoDepacketizer.java +++ b/src/com/limelight/nvstream/av/video/AvVideoDepacketizer.java @@ -13,7 +13,7 @@ import android.media.MediaCodec; public class AvVideoDepacketizer { // Current NAL state - private LinkedList avcNalDataChain = null; + private LinkedList avcNalDataChain = new LinkedList(); private int avcNalDataLength = 0; private int currentlyDecoding; @@ -22,33 +22,33 @@ public class AvVideoDepacketizer { private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(); - private AvByteBufferPool pool = new AvByteBufferPool(1500); + private AvByteBufferPool bbPool = new AvByteBufferPool(1500); public byte[] allocatePacketBuffer() { - return pool.allocate(); + return bbPool.allocate(); } public void trim() { - pool.purge(); + bbPool.purge(); } private void clearAvcNalState() { - if (avcNalDataChain != null) + for (AvByteBufferDescriptor avbb : avcNalDataChain) { - for (AvByteBufferDescriptor avbb : avcNalDataChain) - { - AvVideoPacket packet = (AvVideoPacket) avbb.context; - - if (packet.release() == 0) { - pool.free(avbb.data); - } + AvVideoPacket packet = (AvVideoPacket) avbb.context; + + if (packet.release() == 0) { + bbPool.free(avbb.data); + packet.free(); } + + avbb.free(); } - avcNalDataChain = null; + avcNalDataChain.clear(); avcNalDataLength = 0; } @@ -60,15 +60,20 @@ public class AvVideoDepacketizer { AvVideoPacket packet = (AvVideoPacket) buff.context; if (packet.release() == 0) { - pool.free(buff.data); + bbPool.free(buff.data); + packet.free(); } + + buff.free(); } + + decodeUnit.free(); } private void reassembleAvcNal() { // This is the start of a new NAL - if (avcNalDataChain != null && avcNalDataLength != 0) + if (!avcNalDataChain.isEmpty() && avcNalDataLength != 0) { int flags = 0; @@ -108,6 +113,8 @@ public class AvVideoDepacketizer { header.data[header.offset+4]); break; } + + specialSeq.free(); } else { @@ -118,14 +125,14 @@ public class AvVideoDepacketizer { } // Construct the H264 decode unit - AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, flags); + AvDecodeUnit du = AvDecodeUnit.newDecodeUnit(AvDecodeUnit.TYPE_H264, avcNalDataChain, avcNalDataLength, flags); if (!decodedUnits.offer(du)) { releaseDecodeUnit(du); } // Clear old state - avcNalDataChain = null; + avcNalDataChain.clear(); avcNalDataLength = 0; } } @@ -158,7 +165,7 @@ public class AvVideoDepacketizer { reassembleAvcNal(); // Setup state for the new NAL - avcNalDataChain = new LinkedList(); + avcNalDataChain.clear(); avcNalDataLength = 0; } @@ -210,9 +217,10 @@ public class AvVideoDepacketizer { location.length--; } - if (currentlyDecoding == AvDecodeUnit.TYPE_H264 && avcNalDataChain != null) + if (currentlyDecoding == AvDecodeUnit.TYPE_H264) { - AvByteBufferDescriptor data = new AvByteBufferDescriptor(location.data, start, location.offset-start); + // This is release if the NAL is cleared or decoded + AvByteBufferDescriptor data = AvByteBufferDescriptor.newDescriptor(location.data, start, location.offset-start); // Attach the current packet as the buffer context and increment the refcount data.context = packet; @@ -226,8 +234,12 @@ public class AvVideoDepacketizer { // If nothing useful came out of this, release the packet now if (packet.release() == 0) { - pool.free(location.data); + bbPool.free(location.data); + packet.free(); } + + // Done with the buffer descriptor + location.free(); } public void addInputData(AvRtpPacket packet) @@ -248,9 +260,9 @@ public class AvVideoDepacketizer { lastSequenceNumber = seq; - // Pass the payload to the non-sequencing parser + // Pass the payload to the non-sequencing parser. It now owns that descriptor. AvByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); - addInputData(new AvVideoPacket(rtpPayload)); + addInputData(AvVideoPacket.createNoCopy(rtpPayload)); } public AvDecodeUnit getNextDecodeUnit() throws InterruptedException @@ -304,19 +316,19 @@ class NAL { buffer.data[buffer.offset+3] == 0x01) { // It's the AVC start sequence 00 00 00 01 - return new AvByteBufferDescriptor(buffer.data, buffer.offset, 4); + return AvByteBufferDescriptor.newDescriptor(buffer.data, buffer.offset, 4); } else { // It's 00 00 00 - return new AvByteBufferDescriptor(buffer.data, buffer.offset, 3); + return AvByteBufferDescriptor.newDescriptor(buffer.data, buffer.offset, 3); } } else if (buffer.data[buffer.offset+2] == 0x01 || buffer.data[buffer.offset+2] == 0x02) { // These are easy: 00 00 01 or 00 00 02 - return new AvByteBufferDescriptor(buffer.data, buffer.offset, 3); + return AvByteBufferDescriptor.newDescriptor(buffer.data, buffer.offset, 3); } else if (buffer.data[buffer.offset+2] == 0x03) { @@ -338,7 +350,7 @@ class NAL { else { // It's not a standard replacement so it's a special sequence - return new AvByteBufferDescriptor(buffer.data, buffer.offset, 3); + return AvByteBufferDescriptor.newDescriptor(buffer.data, buffer.offset, 3); } } } diff --git a/src/com/limelight/nvstream/av/video/AvVideoPacket.java b/src/com/limelight/nvstream/av/video/AvVideoPacket.java index fde7bc54..3aa751a9 100644 --- a/src/com/limelight/nvstream/av/video/AvVideoPacket.java +++ b/src/com/limelight/nvstream/av/video/AvVideoPacket.java @@ -1,19 +1,34 @@ package com.limelight.nvstream.av.video; import com.limelight.nvstream.av.AvByteBufferDescriptor; +import com.limelight.nvstream.av.AvObjectPool; public class AvVideoPacket { private AvByteBufferDescriptor buffer; private int refCount; - public AvVideoPacket(AvByteBufferDescriptor rtpPayload) + private static AvObjectPool pool = new AvObjectPool(); + + public static AvVideoPacket createNoCopy(AvByteBufferDescriptor payload) { + AvVideoPacket pkt = pool.tryAllocate(); + if (pkt != null) { + pkt.buffer = payload; + pkt.refCount = 0; + return pkt; + } + else { + return new AvVideoPacket(payload); + } + } + + private AvVideoPacket(AvByteBufferDescriptor rtpPayload) { - buffer = new AvByteBufferDescriptor(rtpPayload); + buffer = rtpPayload; } public AvByteBufferDescriptor getNewPayloadDescriptor() { - return new AvByteBufferDescriptor(buffer.data, buffer.offset+56, buffer.length-56); + return AvByteBufferDescriptor.newDescriptor(buffer.data, buffer.offset+56, buffer.length-56); } public int addRef() @@ -25,4 +40,10 @@ public class AvVideoPacket { { return --refCount; } + + public void free() + { + buffer.free(); + pool.free(this); + } }