Recording Audio and Video using MediaMuxer on Android
I'm trying to record audio and video using AudioRecord, MediaCodec and MediaMuxer provided in Android 4.3
However, sometimes the audio encoder thread stops and is not encoding anymore. The result is, a broken mp4 file, because the muxer does not receive any encoded audio frames. On my Samsung Galaxy Note 3 it is working 99% but on my Sony Xperia Z1 the encoding thread is always stuck. I really don't know what is the reason, maybe someone could help me optimizing my code:
AudioRecorder.java
package com.cmdd.horicam;
import java.nio.ByteBuffer;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaRecorder;
import android.os.Looper;
import android.util.Log;
public class AudioRecorder implements Runnable {
public static final String TAG = "AudioRecorder";
public static final boolean VERBOSE = false;
public MovieMuxerAudioHandler mAudioHandler;
// audio format settings
public static final String MIME_TYPE_AUDIO = "audio/mp4a-latm";
public static final int SAMPLE_RATE = 44100;
public static final int CHANNEL_COUNT = 1;
public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO;
public static final int BIT_RATE_AUDIO = 128000;
public static final int SAMPLES_PER_FRAME = 1024; // AAC
public static final int FRAMES_PER_BUFFER = 24;
public static final int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT;
public static final int AUDIO_SOURCE = MediaRecorder.AudioSource.MIC;
public static final int MSG_START_RECORDING = 0;
public static final int MSG_STOP_RECORDING = 1;
public static final int MSG_QUIT = 2;
private MediaCodec mAudioEncoder;
private int iBufferSize;
int iReadResult = 0;
private boolean bIsRecording = false;
private static final int TIMEOUT_USEC = 10000;
private MovieMuxer mMovieMuxer;
private MediaFormat mAudioFormat;
private volatile AudioRecorderHandler mHandler;
private Object mReadyFence = new Object(); // guards ready/running
private boolean mReady;
private boolean mRunning;
public AudioRecorder(MovieMuxer mMovieMuxer){
this.mMovieMuxer = mMovieMuxer;
}
/**
* Recorder thread entry point. Establishes Looper/Handler and waits for messages.
* <p>
* @see java.lang.Thread#run()
*/
@Override
public void run() {
// Establish a Looper for this thread, and define a Handler for it.
Looper.prepare();
synchronized (mReadyFence) {
mHandler = new AudioRecorderHandler(this);
mReady = true;
mReadyFence.notify();
}
Looper.loop();
if(VERBOSE)Log.d(TAG, "audio recorder exiting thread");
synchronized (mReadyFence) {
mReady = mRunning = false;
mHandler = null;
}
}
public void prepareEncoder(){
// prepare audio format
mAudioFormat = MediaFormat.createAudioFormat(MIME_TYPE_AUDIO, SAMPLE_RATE, CHANNEL_COUNT);
mAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
mAudioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16384);
mAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE_AUDIO);
mAudioEncoder = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
mAudioEncoder.configure(mAudioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mAudioEncoder.start();
new Thread(new AudioEncoderTask(), "AudioEncoderTask").start();
}
public void prepareRecorder() {
int iMinBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT);
bIsRecording = false;
iBufferSize = SAMPLES_PER_FRAME * FRAMES_PER_BUFFER;
// Ensure buffer is adequately sized for the AudioRecord
// object to initialize
if (iBufferSize < iMinBufferSize)
iBufferSize = ((iMinBufferSize / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2;
AudioRecord mAudioRecorder;
mAudioRecorder = new AudioRecord(
AUDIO_SOURCE, // source
SAMPLE_RATE, // sample rate, hz
CHANNEL_CONFIG, // channels
AUDIO_FORMAT, // audio format
iBufferSize); // buffer size (bytes)
mAudioRecorder.startRecording();
new Thread(new AudioRecorderTask(mAudioRecorder), "AudioRecorderTask").start();
}
/**
* Tells the audio recorder to start recording. (Call from non-encoder thread.)
* <p>
* Creates a new thread, which will create an encoder using the provided configuration.
* <p>
* Returns after the recorder thread has started and is ready to accept Messages. The
* encoder may not yet be fully configured.
*/
public void startRecording() {
if(VERBOSE)Log.d(TAG, "audio recorder: startRecording()");
synchronized (mReadyFence) {
if (mRunning) {
Log.w(TAG, "audio recorder thread already running");
return;
}
mRunning = true;
new Thread(this, "AudioRecorder").start();
while (!mReady) {
try {
mReadyFence.wait();
} catch (InterruptedException ie) {
// ignore
}
}
}
mHandler.sendMessage(mHandler.obtainMessage(MSG_START_RECORDING));
}
public void handleStartRecording(){
if(VERBOSE)Log.d(TAG, "handleStartRecording");
prepareEncoder();
prepareRecorder();
bIsRecording = true;
}
/**
* Tells the video recorder to stop recording. (Call from non-encoder thread.)
* <p>
* Returns immediately; the encoder/muxer may not yet be finished creating the movie.
* <p>
*/
public void stopRecording() {
if(mHandler != null){
mHandler.sendMessage(mHandler.obtainMessage(MSG_STOP_RECORDING));
mHandler.sendMessage(mHandler.obtainMessage(MSG_QUIT));
}
}
/**
* Handles a request to stop encoding.
*/
public void handleStopRecording() {
if(VERBOSE)Log.d(TAG, "handleStopRecording");
bIsRecording = false;
}
public String getCurrentAudioFormat(){
if(this.mAudioFormat == null)
return "null";
else
return this.mAudioFormat.toString();
}
private class AudioRecorderTask implements Runnable {
AudioRecord mAudioRecorder;
ByteBuffer[] inputBuffers;
ByteBuffer inputBuffer;
public AudioRecorderTask(AudioRecord recorder){
this.mAudioRecorder = recorder;
}
@Override
public void run() {
if(VERBOSE)Log.i(TAG, "AudioRecorder started recording");
long audioPresentationTimeNs;
byte[] mTempBuffer = new byte[SAMPLES_PER_FRAME];
while (bIsRecording) {
audioPresentationTimeNs = System.nanoTime();
iReadResult = mAudioRecorder.read(mTempBuffer, 0, SAMPLES_PER_FRAME);
if(iReadResult == AudioRecord.ERROR_BAD_VALUE || iReadResult == AudioRecord.ERROR_INVALID_OPERATION)
Log.e(TAG, "audio buffer read error");
// send current frame data to encoder
try {
if(inputBuffers == null)
inputBuffers = mAudioEncoder.getInputBuffers();
int inputBufferIndex = mAudioEncoder.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(mTempBuffer);
//recycleInputBuffer(mTempBuffer);
if(VERBOSE)Log.d(TAG, "sending frame to audio encoder");
mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, mTempBuffer.length, audioPresentationTimeNs / 1000, 0);
}
} catch (Throwable t) {
Log.e(TAG, "sendFrameToAudioEncoder exception");
t.printStackTrace();
}
}
// finished recording -> send it to the encoder
audioPresentationTimeNs = System.nanoTime();
iReadResult = mAudioRecorder.read(mTempBuffer, 0, SAMPLES_PER_FRAME);
if (iReadResult == AudioRecord.ERROR_BAD_VALUE
|| iReadResult == AudioRecord.ERROR_INVALID_OPERATION)
Log.e(TAG, "audio buffer read error");
// send current frame data to encoder
try {
int inputBufferIndex = mAudioEncoder.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(mTempBuffer);
if(VERBOSE)Log.d(TAG, "sending EOS to audio encoder");
mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, mTempBuffer.length, audioPresentationTimeNs / 1000, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
} catch (Throwable t) {
Log.e(TAG, "sendFrameToAudioEncoder exception");
t.printStackTrace();
}
//if (mAudioRecorder != null) {
// mAudioRecorder.release();
// mAudioRecorder = null;
// if(VERBOSE)Log.i(TAG, "stopped");
//}
}
}
private class AudioEncoderTask implements Runnable {
private boolean bAudioEncoderFinished;
private int iAudioTrackIndex;
private MediaCodec.BufferInfo mAudioBufferInfo;
@Override
public void run(){
if(VERBOSE)Log.i(TAG, "AudioEncoder started encoding");
bAudioEncoderFinished = false;
ByteBuffer[] encoderOutputBuffers = mAudioEncoder.getOutputBuffers();
ByteBuffer encodedData;
mAudioBufferInfo = new MediaCodec.BufferInfo();
while(!bAudioEncoderFinished){
int encoderStatus = mAudioEncoder.dequeueOutputBuffer(mAudioBufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG + "_encoder", "no output available, spinning to await EOS");
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
encoderOutputBuffers = mAudioEncoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = mAudioEncoder.getOutputFormat();
if(VERBOSE)Log.d(TAG, "received output format: " + newFormat);
// should happen before receiving buffers, and should only happen once
iAudioTrackIndex = mMovieMuxer.addTrack(newFormat);
} else if (encoderStatus < 0) {
Log.w(TAG + "_encoder", "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
// let's ignore it
} else {
if(mMovieMuxer.muxerStarted()){
encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null) {
throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null");
}
if ((mAudioBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
if (VERBOSE) Log.d(TAG + "_encoder", "ignoring BUFFER_FLAG_CODEC_CONFIG");
mAudioBufferInfo.size = 0;
}
if (mAudioBufferInfo.size != 0) {
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(mAudioBufferInfo.offset);
encodedData.limit(mAudioBufferInfo.offset + mAudioBufferInfo.size);
mMovieMuxer.writeSampleData(iAudioTrackIndex, encodedData, mAudioBufferInfo);
if (VERBOSE) {
Log.d(TAG + "_encoder", "sent " + mAudioBufferInfo.size + " bytes (audio) to muxer, ts=" + mAudioBufferInfo.presentationTimeUs);
}
}
mAudioEncoder.releaseOutputBuffer(encoderStatus, false);
if ((mAudioBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
// reached EOS
if(VERBOSE)Log.i(TAG + "_encoder", "audio encoder finished");
bAudioEncoderFinished = true;
// tell the muxer that we are finished
mAudioHandler.onAudioEncodingFinished();
break;
}
}
}
}
}
}
}
Thanks for your help.
Answer
When you requesting data from audio record :
iReadResult = mAudioRecorder.read(mTempBuffer, 0, SAMPLES_PER_FRAME);
You could obtain several frames, then pts predictor in mediacodec will generate proper output pts based on number of frames and compressed frame duration. Then you can print those timestamp after encoder dequeueoutputbuffer and see actual value will be !0. But then you will feed encoder again with 0 pts at input, and it will reset internal prediction. This all wil result in non monotonic pts generation, and proably muxer already complains for that, check in adb logs. For me that was happened and i have to set Sample time maually before feeding encoder.
mTempBuffer.setSampleTime(calc_pts_for_that_frame);
At least you can check if this is an issue you are facing with, and if so it is easy to solve by calculate appropiate timestamps.