Skip to content

armgit / haven

Public Imported from GitHub
0

Haven is for people who need a way to protect their personal spaces and possessions without compromising their own privacy, through an Android app and on-device sensors

begin work on hardware video encoder using MediaCodec

n8fr877fd35f21fca
diff --git a/src/main/java/org/havenapp/main/sensors/media/VideoEncoder.java b/src/main/java/org/havenapp/main/sensors/media/VideoEncoder.java
new file mode 100644
index 0000000..b88854c
--- /dev/null
+++ b/src/main/java/org/havenapp/main/sensors/media/VideoEncoder.java
@@ -0,0 +1,488 @@
+package org.havenapp.main.sensors.media;
+
+
+import android.annotation.SuppressLint;
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaCodecList;
+import android.media.MediaFormat;
+import android.media.MediaMuxer;
+import android.os.Build;
+import android.support.annotation.RequiresApi;
+import android.util.Log;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/**
+ * Generates a series of video frames, encodes them, decodes them, and tests for
+ * significant divergence from the original.
+ */
+public class VideoEncoder {
+
+ private static final String TAG = "EncodeDecode";
+ private static final boolean VERBOSE = false; // lots of logging
+ // parameters for the encoder
+ private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video
+ // Coding
+ private static final int FRAME_RATE = 10; // 10fps
+ private static final int IFRAME_INTERVAL = 10; // 10 seconds between
+ // I-frames
+ // size of a frame, in pixels
+ private int mWidth = -1;
+ private int mHeight = -1;
+ // bit rate, in bits per second
+ private int mBitRate = -1;
+ // largest color component delta seen (i.e. actual vs. expected)
+ private int mLargestColorDelta;
+
+ private File outputFile = null;
+ private MediaCodec mEncoder;
+ private MediaMuxer mMuxer;
+ private int mTrackIndex;
+ private boolean mMuxerStarted;
+ private ArrayList<File> frames;
+
+ public VideoEncoder(ArrayList<File> frames, File outputFile)
+ {
+ this.frames = frames;
+ this.outputFile = outputFile;
+ }
+
+ /**
+ * Tests streaming of AVC video through the encoder and decoder. Data is
+ * encoded from a series of byte[] buffers and decoded into Surfaces. The
+ * output is checked for validity.
+ */
+ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
+ public boolean encodeDecodeVideoFromBufferToSurface(int width, int height,
+ int bitRate) throws Throwable
+ {
+ setParameters(width, height, bitRate);
+ return encodeDecodeVideoFromBuffer();
+ }
+
+ /**
+ * Sets the desired frame size and bit rate.
+ */
+ private void setParameters(int width, int height, int bitRate)
+ {
+ if ((width % 16) != 0 || (height % 16) != 0)
+ {
+ Log.w(TAG, "WARNING: width or height not multiple of 16");
+ }
+ mWidth = width;
+ mHeight = height;
+ mBitRate = bitRate;
+ }
+
+ /**
+ * Tests encoding and subsequently decoding video from frames generated into
+ * a buffer.
+ */
+ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
+ @SuppressLint("InlinedApi")
+ public boolean encodeDecodeVideoFromBuffer()
+ throws Exception
+ {
+ mLargestColorDelta = -1;
+ boolean result = true;
+ try
+ {
+ MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
+ if (codecInfo == null)
+ {
+ // Don't fail CTS if they don't have an AVC codec
+ Log.e(TAG, "Unable to find an appropriate codec for "
+ + MIME_TYPE);
+ return false;
+ }
+ if (VERBOSE)
+ Log.d(TAG, "found codec: " + codecInfo.getName());
+ int colorFormat;
+ try
+ {
+ colorFormat = selectColorFormat(codecInfo, MIME_TYPE);
+ } catch (Exception e)
+ {
+ colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
+ }
+ if (VERBOSE)
+ Log.d(TAG, "found colorFormat: " + colorFormat);
+ // We avoid the device-specific limitations on width and height by
+ // using values that
+ // are multiples of 16, which all tested devices seem to be able to
+ // handle.
+ MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE,
+ mWidth, mHeight);
+ // Set some properties. Failing to specify some of these can cause
+ // the MediaCodec
+ // configure() call to throw an unhelpful exception.
+ format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
+ format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate);
+ format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
+ format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
+ if (VERBOSE)
+ Log.d(TAG, "format: " + format);
+ // Create a MediaCodec for the desired codec, then configure it as
+ // an encoder with
+ // our desired properties.
+ mEncoder = MediaCodec.createByCodecName(codecInfo.getName());
+ mEncoder.configure(format, null, null,
+ MediaCodec.CONFIGURE_FLAG_ENCODE);
+ mEncoder.start();
+ // Create a MediaCodec for the decoder, just based on the MIME type.
+ // The various
+ // format details will be passed through the csd-0 meta-data later
+ // on.
+ String outputPath = outputFile.getAbsolutePath();
+ try
+ {
+ mMuxer = new MediaMuxer(outputPath,
+ MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
+ } catch (IOException ioe)
+ {
+ // throw new RuntimeException("MediaMuxer creation failed",
+ // ioe);
+ ioe.printStackTrace();
+ }
+ result = doEncodeDecodeVideoFromBuffer(mEncoder, colorFormat);
+ } finally
+ {
+ if (mEncoder != null)
+ {
+ mEncoder.stop();
+ mEncoder.release();
+ }
+ if (mMuxer != null)
+ {
+ mMuxer.stop();
+ mMuxer.release();
+ }
+ if (VERBOSE)
+ Log.i(TAG, "Largest color delta: " + mLargestColorDelta);
+ }
+ return result;
+ }
+
+ /**
+ * Returns the first codec capable of encoding the specified MIME type, or
+ * null if no match was found.
+ */
+ private static MediaCodecInfo selectCodec(String mimeType)
+ {
+ int numCodecs = MediaCodecList.getCodecCount();
+ for (int i = 0; i < numCodecs; i++)
+ {
+ MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
+ if (!codecInfo.isEncoder())
+ {
+ continue;
+ }
+ String[] types = codecInfo.getSupportedTypes();
+ for (int j = 0; j < types.length; j++)
+ {
+ if (types[j].equalsIgnoreCase(mimeType))
+ {
+ return codecInfo;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns a color format that is supported by the codec and by this test
+ * code. If no match is found, this throws a test failure -- the set of
+ * formats known to the test should be expanded for new platforms.
+ */
+ private static int selectColorFormat(MediaCodecInfo codecInfo,
+ String mimeType)
+ {
+ MediaCodecInfo.CodecCapabilities capabilities = codecInfo
+ .getCapabilitiesForType(mimeType);
+ for (int i = 0; i < capabilities.colorFormats.length; i++)
+ {
+ int colorFormat = capabilities.colorFormats[i];
+ if (isRecognizedFormat(colorFormat))
+ {
+ return colorFormat;
+ }
+ }
+ return 0; // not reached
+ }
+
+ /**
+ * Returns true if this is a color format that this test code understands
+ * (i.e. we know how to read and generate frames in this format).
+ */
+ private static boolean isRecognizedFormat(int colorFormat)
+ {
+ switch (colorFormat)
+ {
+ // these are the formats we know how to handle for
+ case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
+ case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
+ case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
+ case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
+ case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Does the actual work for encoding frames from buffers of byte[].
+ */
+ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
+ @SuppressLint("InlinedApi")
+ private boolean doEncodeDecodeVideoFromBuffer(MediaCodec encoder,
+ int encoderColorFormat)
+ {
+ final int TIMEOUT_USEC = 10000;
+ ByteBuffer[] encoderInputBuffers = encoder.getInputBuffers();
+ MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
+ int generateIndex = 0;
+ // yuv format
+ byte[] frameData = new byte[mWidth * mHeight * 3 / 2];
+ // Loop until the output side is done.
+ boolean inputDone = false;
+ // If we're not done submitting frames, generate a new one and submit
+ // it. By
+ // doing this on every loop we're working to ensure that the encoder
+ // always has
+ // work to do.
+ while (!inputDone)
+ {
+ int inputBufIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
+ if (inputBufIndex >= 0)
+ {
+ long ptsUsec = computePresentationTime(generateIndex);
+ if (generateIndex >= frames.size())
+ {
+ // Send an empty frame with the end-of-stream flag set. If
+ // we set EOS
+ // on a frame with data, that frame data will be ignored,
+ // and the
+ // output will be short one frame.
+ encoder.queueInputBuffer(inputBufIndex, 0, 0, ptsUsec,
+ MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+ inputDone = true;
+ drainEncoder(true, info);
+ } else
+ {
+ try
+ {
+ generateFrame(generateIndex, encoderColorFormat,
+ frameData);
+ } catch (Exception e)
+ {
+ Log.i(TAG, "meet a different type of image");
+ Arrays.fill(frameData, (byte) 0);
+ }
+ if (VERBOSE)
+ Log.i(TAG, "generateIndex: " + generateIndex
+ + ", size: " + frames.size());
+ ByteBuffer inputBuf = encoderInputBuffers[inputBufIndex];
+ // the buffer should be sized to hold one full frame
+ inputBuf.clear();
+ inputBuf.put(frameData);
+ encoder.queueInputBuffer(inputBufIndex, 0,
+ frameData.length, ptsUsec, 0);
+ drainEncoder(false, info);
+ }
+ generateIndex++;
+ } else
+ {
+ // either all in use, or we timed out during initial setup
+ if (VERBOSE)
+ Log.i(TAG, "input buffer not available");
+ }
+ }
+ return true;
+ }
+
+ /**
+ * use Muxer to generate mp4 file with data from encoder
+ *
+ * @param endOfStream
+ * if this is the last frame
+ * @param mBufferInfo
+ * the BufferInfo of data from encoder
+ */
+ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
+ private void drainEncoder(boolean endOfStream, MediaCodec.BufferInfo mBufferInfo)
+ {
+ final int TIMEOUT_USEC = 10000;
+
+ if (endOfStream)
+ {
+ try
+ {
+ mEncoder.signalEndOfInputStream();
+ } catch (Exception e)
+ {
+ }
+ }
+
+ ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
+ while (true)
+ {
+ int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo,
+ TIMEOUT_USEC);
+ if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
+ {
+ // no output available yet
+ if (!endOfStream)
+ {
+ break; // out of while
+ } else
+ {
+ if (VERBOSE)
+ Log.i(TAG, "no output available, spinning to await EOS");
+ }
+ } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
+ {
+ // not expected for an encoder
+ encoderOutputBuffers = mEncoder.getOutputBuffers();
+ } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
+ {
+ // should happen before receiving buffers, and should only
+ // happen once
+ if (mMuxerStarted)
+ {
+ throw new RuntimeException("format changed twice");
+ }
+ MediaFormat newFormat = mEncoder.getOutputFormat();
+ if (VERBOSE)
+ Log.i(TAG, "encoder output format changed: " + newFormat);
+
+ // now that we have the Magic Goodies, start the muxer
+ mTrackIndex = mMuxer.addTrack(newFormat);
+ mMuxer.start();
+ mMuxerStarted = true;
+ } else if (encoderStatus < 0)
+ {
+ if (VERBOSE)
+ Log.i(TAG,
+ "unexpected result from encoder.dequeueOutputBuffer: "
+ + encoderStatus);
+ } else
+ {
+ ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
+ if (encodedData == null)
+ {
+ throw new RuntimeException("encoderOutputBuffer "
+ + encoderStatus + " was null");
+ }
+
+ if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0)
+ {
+ // The codec config data was pulled out and fed to the muxer
+ // when we got
+ // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
+ if (VERBOSE)
+ Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
+ mBufferInfo.size = 0;
+ }
+
+ if (mBufferInfo.size != 0)
+ {
+ if (!mMuxerStarted)
+ {
+ throw new RuntimeException("muxer hasn't started");
+ }
+
+ // adjust the ByteBuffer values to match BufferInfo
+ encodedData.position(mBufferInfo.offset);
+ encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
+
+ if (VERBOSE)
+ Log.d(TAG, "BufferInfo: " + mBufferInfo.offset + ","
+ + mBufferInfo.size + ","
+ + mBufferInfo.presentationTimeUs);
+
+ try
+ {
+ mMuxer.writeSampleData(mTrackIndex, encodedData,
+ mBufferInfo);
+ } catch (Exception e)
+ {
+ Log.i(TAG, "Too many frames");
+ }
+ }
+
+ mEncoder.releaseOutputBuffer(encoderStatus, false);
+
+ if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0)
+ {
+ if (!endOfStream)
+ {
+ if (VERBOSE)
+ Log.i(TAG, "reached end of stream unexpectedly");
+ } else
+ {
+ if (VERBOSE)
+ Log.i(TAG, "end of stream reached");
+ }
+ break; // out of while
+ }
+ }
+ }
+ }
+
+ /**
+ * Generates data for frame N into the supplied buffer.
+ */
+ private void generateFrame(int frameIndex, int colorFormat, byte[] frameData)
+ {
+ // Set to zero. In YUV this is a dull green.
+ Arrays.fill(frameData, (byte) 0);
+
+ /**
+ Mat mat = Highgui.imread(frames.get(frameIndex).getAbsolutePath());
+
+// Mat dst = new Mat(mWidth, mHeight * 3 / 2, CvType.CV_8UC1);
+ Mat dst = new Mat();
+ Imgproc.cvtColor(mat, dst, Imgproc.COLOR_RGBA2YUV_I420);
+
+ // use array instead of mat to improve the speed
+ dst.get(0, 0, frameData);
+
+ byte[] temp = frameData.clone();
+ int margin = mHeight / 4;
+ int location = mHeight;
+ int step = 0;
+ for (int i = mHeight; i < mHeight + margin; i++)
+ {
+ for (int j = 0; j < mWidth; j++)
+ {
+ byte uValue = temp[i * mWidth + j];
+ byte vValue = temp[(i + margin) * mWidth + j];
+
+ frameData[location * mWidth + step] = uValue;
+ frameData[location * mWidth + step + 1] = vValue;
+ step += 2;
+ if (step >= mWidth)
+ {
+ location++;
+ step = 0;
+ }
+ }
+ }
+ **/
+ }
+
+ /**
+ * Generates the presentation time for frame N, in microseconds.
+ */
+ private static long computePresentationTime(int frameIndex)
+ {
+ long value = frameIndex;
+ return 132 + value * 1000000 / FRAME_RATE;
+ }
+}
\ No newline at end of file