Android 音视频合成经验总结

刚刚实现了音视频合成的需求，趁热打铁记录一下遇到的问题

需求描述：

将给定的音频及视频合成一个视频，如果音频时长小于视频，则重复播放；如果大于视频时长则截取到视频结束。

采用的是MediaMuxer、MediaCodec原生方案。

MediaCodec主要处理音频格式的问题，音频可能有mp3 m4a等一些格式，解析得到的音频编码有一个是安卓不支持的编码格式，如audio/mpeg，添加音轨时就会抛出异常。所以这里统一用MediaCodec解析成audio/aac格式，解析方法参考了网上的解析

    /**
     *
     * @param audioPath
     * @param audioStartTimeUs -1 表示不截取
     * @param audioEndTimeUs -1 表示不截取
     * @return
     */
    public String audioToAAC(String audioPath,long audioStartTimeUs,long audioEndTimeUs){

        long a=System.currentTimeMillis();
        int audioExtractorTrackIndex=-1;
        int audioMuxerTrackIndex=-1;
        int channelCount=1;
        int sourceSampleRate = 0;
        String newAudioAAc = "";
        long sourceDuration=0;

        try {
            File tempFlie = new File(audioPath);
            String tempName = tempFlie.getName();
            String suffix = tempName.substring(tempName.lastIndexOf(".") + 1);
//             newAudioAAc= tempFlie.getParentFile().getAbsolutePath() + "/" + tempName.replace(suffix, "aac");
            newAudioAAc= Objects.requireNonNull(tempFlie.getParentFile()).getAbsolutePath() + "/" + tempName.replace(suffix, "aac");
//            muxer = new MediaMuxer(newAudioAAc, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
            Log.i(TAG,"audioPath=="+audioPath+",newAudioAAC=="+newAudioAAc+",newAudioAAc=="+newAudioAAc);
            //音频信息获取
            MediaExtractor audioExtractor = new MediaExtractor();
            audioExtractor.setDataSource(audioPath);
            int trackCount = audioExtractor.getTrackCount();

            MediaFormat sourceFormat=null;
            String sourceMimeType="";
            int timeOutUs=300;

            for (int i = 0; i < trackCount; i++) {
                sourceFormat = audioExtractor.getTrackFormat(i);
                sourceMimeType = sourceFormat.getString(MediaFormat.KEY_MIME);
                channelCount=sourceFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
                sourceSampleRate = sourceFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
                sourceDuration = sourceFormat.getLong(MediaFormat.KEY_DURATION);
                if (sourceMimeType.startsWith("audio/")) { //找到音轨
                    Log.i(TAG,"sourceMimeType=="+sourceMimeType+",channelCount=="+channelCount+",sourceSampleRate=="+sourceSampleRate);
                    audioExtractorTrackIndex = i;
                    break;
                }
            }

            //初始化解码器
            MediaCodec audioDecoder = null;
            audioDecoder = MediaCodec.createDecoderByType(sourceMimeType);
            audioDecoder.configure(sourceFormat, null, null, 0);
            audioDecoder.start();

            //初始化编码
            MediaCodec mEncorder = null;
            mEncorder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
            MediaFormat format = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, 44100, channelCount);
            format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) ;
            format.setInteger(MediaFormat.KEY_BIT_RATE, 128000);
            format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 1024*1024 * 10);
            mEncorder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
            mEncorder.start();

            audioExtractor.selectTrack(audioExtractorTrackIndex);

            MediaCodec.BufferInfo sourceAudioBufferInfo = new MediaCodec.BufferInfo();
            MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();

            ByteBuffer audioByteBuffer = ByteBuffer.allocate(1024*1024*10);
            FileOutputStream mFileStream = new FileOutputStream(newAudioAAc);
  
            while (true) {
                int readSampleSize = audioExtractor.readSampleData(audioByteBuffer, 0);
                Log.i(TAG, "readSampleSize==" + readSampleSize+",audioByteBuffer.limit=="+audioByteBuffer.limit()+",timeOutUs=="+timeOutUs);
                if (readSampleSize < 0) {
                    audioExtractor.unselectTrack(audioExtractorTrackIndex);
                    break;
                }

                long audioSampleTime=audioExtractor.getSampleTime();

                //可以做进度回调
                Log.i(TAG, "audioSampleTime==" +audioSampleTime+",,progress=="+((float)audioSampleTime/(float)sourceDuration));
                if (audioStartTimeUs !=-1 && audioSampleTime < audioStartTimeUs) {
                    audioExtractor.advance();
                    continue;
                }

                if (audioEndTimeUs !=-1 && audioSampleTime > audioEndTimeUs) {
                    break;
                }

                int audioSampleFlags=audioExtractor.getSampleFlags();

                //解码
                int sourceInputBufferIndex = audioDecoder.dequeueInputBuffer(timeOutUs);
                if (sourceInputBufferIndex >= 0) {
                    ByteBuffer sourceInputBuffer = audioDecoder.getInputBuffer(sourceInputBufferIndex);
                    sourceInputBuffer.clear();
                    sourceInputBuffer.put(audioByteBuffer);
                    audioDecoder.queueInputBuffer(sourceInputBufferIndex, 0, readSampleSize, audioSampleTime, audioSampleFlags);
                }


                int sourceOutputBufferIndex = audioDecoder.dequeueOutputBuffer(sourceAudioBufferInfo, timeOutUs);
                if (sourceOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                    // 后续输出格式变化
                }
                while (sourceOutputBufferIndex >= 0) {
                    ByteBuffer decoderOutputBuffer = audioDecoder.getOutputBuffer(sourceOutputBufferIndex);
                    
                    //编码
                    int inputBufferIndex = mEncorder.dequeueInputBuffer(timeOutUs);
                    if (inputBufferIndex >= 0) {
                        ByteBuffer inputBuffer = mEncorder.getInputBuffer(inputBufferIndex);
                        inputBuffer.clear();
                        inputBuffer.put(decoderOutputBuffer);
                        mEncorder.queueInputBuffer(inputBufferIndex, 0, decoderOutputBuffer.limit(), audioSampleTime, audioSampleFlags);
                    }

                    int outputBufferIndex = mEncorder.dequeueOutputBuffer(audioBufferInfo, timeOutUs);
                    if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                        // 后续输出格式变化
                    }
                    while (outputBufferIndex >= 0) {

                        ByteBuffer outputBuffer = mEncorder.getOutputBuffer(outputBufferIndex);
                        int outBufferSize = outputBuffer.limit() + 7;
                        byte[] aacBytes = new byte[outBufferSize];
                        addADTStoPacket(aacBytes, outBufferSize, channelCount);
                        outputBuffer.get(aacBytes, 7, outputBuffer.limit());

                        mFileStream.write(aacBytes);

                        mEncorder.releaseOutputBuffer(outputBufferIndex, false);
                        outputBufferIndex = mEncorder.dequeueOutputBuffer(audioBufferInfo, timeOutUs);
                    }




                    audioDecoder.releaseOutputBuffer(sourceOutputBufferIndex, false);
                    sourceOutputBufferIndex = audioDecoder.dequeueOutputBuffer(sourceAudioBufferInfo, timeOutUs);
                }
                audioExtractor.advance();
            }

            //释放资源
            mEncorder.stop();
            mFileStream.flush();
            mFileStream.close();
            long b=System.currentTimeMillis()-a;
            Log.i(TAG,"编码结束=="+b);
        } catch (Exception e) {
            e.printStackTrace();

        }
        return newAudioAAc;
    }

得到aac编码的音频数据后，我们就可以将音视频合入到同一段视频中

    @SuppressLint("WrongConstant")
    public void mix() {
        MediaExtractor videoExtractor = null;
        MediaExtractor audioExtractor = null;
        MediaMuxer mixMediaMuxer = null;
        try {
            videoExtractor = new MediaExtractor();
            videoExtractor.setDataSource(outputVideoFilePath);
            int videoIndex = -1;
            MediaFormat videoTrackFormat = null;
            int trackCount = videoExtractor.getTrackCount();
            for (int i = 0; i < trackCount; i++) {
                videoTrackFormat = videoExtractor.getTrackFormat(i);
                if (Objects.requireNonNull(videoTrackFormat.getString(MediaFormat.KEY_MIME)).startsWith("video/")) {
                    videoIndex = i;
                }
            }

            audioExtractor = new MediaExtractor();
            audioExtractor.setDataSource(outputAudioFilePath);
            int audioIndex = -1;
            MediaFormat audioTrackFormat = null;
            trackCount = audioExtractor.getTrackCount();
            for (int i = 0; i < trackCount; i++) {
                audioTrackFormat = audioExtractor.getTrackFormat(i);
                if (Objects.requireNonNull(audioTrackFormat.getString(MediaFormat.KEY_MIME)).startsWith("audio/")) {
                    audioIndex = i;
                }
            }

            videoExtractor.selectTrack(videoIndex);
            audioExtractor.selectTrack(audioIndex);
            // 初始化视频和音频缓冲区信息
            MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
            MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();
            // 初始化MediaMuxer，用于合成输出文件
            mixMediaMuxer = new MediaMuxer(outputFilePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
            assert videoTrackFormat != null;
            int videoTrackIndex = mixMediaMuxer.addTrack(videoTrackFormat);
            assert audioTrackFormat != null;
            int audioTrackIndex = mixMediaMuxer.addTrack(audioTrackFormat);

            mixMediaMuxer.start();

            ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024);

            long audioTime = 0L;
            long cycleTimes = 0;
            videoExtractor.unselectTrack(videoIndex);
            videoExtractor.selectTrack(videoIndex);

            while (true) {
                int data = videoExtractor.readSampleData(byteBuffer, 0);
                if (data < 0) {
                    break;
                }
                videoBufferInfo.size = data;
                videoBufferInfo.presentationTimeUs = videoExtractor.getSampleTime();
                videoBufferInfo.offset = 0;
                videoBufferInfo.flags = videoExtractor.getSampleFlags();
                mixMediaMuxer.writeSampleData(videoTrackIndex, byteBuffer, videoBufferInfo);
                videoExtractor.advance();
            }


            while (true) {
                int data = audioExtractor.readSampleData(byteBuffer, 0);
                if (data < 0) {
                    if (audioBufferInfo.presentationTimeUs < videoBufferInfo.presentationTimeUs) {
                        if (cycleTimes == 0) {
                            audioTime = audioBufferInfo.presentationTimeUs;
                        }
                        cycleTimes ++;
                        audioExtractor.seekTo(0, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
                        data = audioExtractor.readSampleData(byteBuffer, 0);
                    } else {
                        break;
                    }
                }
                audioBufferInfo.size = data;
                audioBufferInfo.presentationTimeUs = audioTime * cycleTimes + audioExtractor.getSampleTime();
                audioBufferInfo.offset = 0;
                audioBufferInfo.flags = audioExtractor.getSampleFlags();
                mixMediaMuxer.writeSampleData(audioTrackIndex, byteBuffer, audioBufferInfo);
                audioExtractor.advance();
                if (videoBufferInfo.presentationTimeUs <= audioBufferInfo.presentationTimeUs) {
                    break;
                }
            }
            myCallBack.muxerResult(true);
        } catch (IOException e) {
            Log.i("dealBug", "mix exception " + e);
            e.printStackTrace();
            myCallBack.muxerResult(false);
        } finally {
            if (mixMediaMuxer != null) {
                mixMediaMuxer.stop();
                mixMediaMuxer.release();
            }
            videoExtractor.release();
            audioExtractor.release();
        }
    }

这里遇到的坑，原先在网上找了类似的audioTime和videoTime他们采用的策略是获取头两个关键帧取标准值，作为音视频的时间戳。但这样获取会存在不准的问题，advance之后，每次获取到的data值并不是固定的，所以录出的视频播放速度可能存在要么快或者慢的问题。

记录一下，防止下次忘记！