c++ - H.264 muxed to MP4 using libavformat not playing back

Question

I am trying to mux H.264 data into a MP4 file. There appear to be no errors in saving this H.264 Annex B data out to an MP4 file, but the file fails to playback.

I've done a binary comparison on the files and the issue seems to be somewhere in what is being written to the footer (trailer) of the MP4 file.

I suspect it has to be something with the way the stream is being created or something.

Init:

AVOutputFormat* fmt = av_guess_format( 0, "out.mp4", 0 );
oc = avformat_alloc_context();
oc->oformat = fmt;
strcpy(oc->filename, filename);

Part of this prototype app I have is creating a png file for each IFrame. So when the first IFrame is encountered, I create the video stream and write the av header etc:

void addVideoStream(AVCodecContext* decoder)
{
    videoStream = av_new_stream(oc, 0);
    if (!videoStream)
    {
         cout << "ERROR creating video stream" << endl;
         return;        
    }
    vi = videoStream->index;    
    videoContext = videoStream->codec;      
    videoContext->codec_type = AVMEDIA_TYPE_VIDEO;
    videoContext->codec_id = decoder->codec_id;
    videoContext->bit_rate = 512000;
    videoContext->width = decoder->width;
    videoContext->height = decoder->height;
    videoContext->time_base.den = 25;
    videoContext->time_base.num = 1;    
    videoContext->gop_size = decoder->gop_size;
    videoContext->pix_fmt = decoder->pix_fmt;       

    if (oc->oformat->flags & AVFMT_GLOBALHEADER)
        videoContext->flags |= CODEC_FLAG_GLOBAL_HEADER;

    av_dump_format(oc, 0, filename, 1);

    if (!(oc->oformat->flags & AVFMT_NOFILE))
    {
        if (avio_open(&oc->pb, filename, AVIO_FLAG_WRITE) < 0) {
        cout << "Error opening file" << endl;
    }
    avformat_write_header(oc, NULL);
}

I write packets out:

unsigned char* data = block->getData();
unsigned char videoFrameType = data[4];
int dataLen = block->getDataLen();

// store pps
if (videoFrameType == 0x68)
{
    if (ppsFrame != NULL)
    {
        delete ppsFrame; ppsFrameLength = 0; ppsFrame = NULL;
    }
    ppsFrameLength = block->getDataLen();
    ppsFrame = new unsigned char[ppsFrameLength];
    memcpy(ppsFrame, block->getData(), ppsFrameLength);
}
else if (videoFrameType == 0x67)
{
    // sps
    if (spsFrame != NULL)
    {
        delete spsFrame; spsFrameLength = 0; spsFrame = NULL;
}
    spsFrameLength = block->getDataLen();
    spsFrame = new unsigned char[spsFrameLength];
    memcpy(spsFrame, block->getData(), spsFrameLength);                 
}                                           

if (videoFrameType == 0x65 || videoFrameType == 0x41)
{
    videoFrameNumber++;
}
if (videoFrameType == 0x65)
{
    decodeIFrame(videoFrameNumber, spsFrame, spsFrameLength, ppsFrame, ppsFrameLength, data, dataLen);
}

if (videoStream != NULL)
{
    AVPacket pkt = { 0 };
    av_init_packet(&pkt);
    pkt.stream_index = vi;
    pkt.flags = 0;                      
    pkt.pts = pkt.dts = 0;                                  

    if (videoFrameType == 0x65)
    {
        // combine the SPS PPS & I frames together
        pkt.flags |= AV_PKT_FLAG_KEY;                                                   
        unsigned char* videoFrame = new unsigned char[spsFrameLength+ppsFrameLength+dataLen];
        memcpy(videoFrame, spsFrame, spsFrameLength);
        memcpy(&videoFrame[spsFrameLength], ppsFrame, ppsFrameLength);
        memcpy(&videoFrame[spsFrameLength+ppsFrameLength], data, dataLen);

        // overwrite the start code (00 00 00 01 with a 32-bit length)
        setLength(videoFrame, spsFrameLength-4);
        setLength(&videoFrame[spsFrameLength], ppsFrameLength-4);
        setLength(&videoFrame[spsFrameLength+ppsFrameLength], dataLen-4);
        pkt.size = dataLen + spsFrameLength + ppsFrameLength;
        pkt.data = videoFrame;
        av_interleaved_write_frame(oc, &pkt);
        delete videoFrame; videoFrame = NULL;
    }
    else if (videoFrameType != 0x67 && videoFrameType != 0x68)
    {   
        // Send other frames except pps & sps which are caught and stored                   
        pkt.size = dataLen;
        pkt.data = data;
        setLength(data, dataLen-4);                     
        av_interleaved_write_frame(oc, &pkt);
    }

Finally to close the file off:

av_write_trailer(oc);
int i = 0;
for (i = 0; i < oc->nb_streams; i++)
{
    av_freep(&oc->streams[i]->codec);
    av_freep(&oc->streams[i]);      
}

if (!(oc->oformat->flags & AVFMT_NOFILE))
{
    avio_close(oc->pb);
}
av_free(oc);

If I take the H.264 data alone and convert it:

ffmpeg -i recording.h264 -vcodec copy recording.mp4

All but the "footer" of the files are the same.

Output from my program: readrec recording.tcp out.mp4 **** START **** 01-03-2013 14:26:01 180000 Output #0, mp4, to 'out.mp4': Stream #0:0: Video: h264, yuv420p, 352x288, q=2-31, 512 kb/s, 90k tbn, 25 tbc **** END **** 01-03-2013 14:27:01 102000 Wrote 1499 video frames.

If I try to convert using ffmpeg the MP4 file created using CODE:

ffmpeg -i out.mp4 -vcodec copy out2.mp4
ffmpeg version 0.11.1 Copyright (c) 2000-2012 the FFmpeg developers
      built on Mar  7 2013 12:49:22 with suncc 0x5110
      configuration: --extra-cflags=-KPIC -g --disable-mmx
      --disable-protocol=udp --disable-encoder=nellymoser --cc=cc --cxx=CC
libavutil      51. 54.100 / 51. 54.100
libavcodec     54. 23.100 / 54. 23.100
libavformat    54.  6.100 / 54.  6.100
libavdevice    54.  0.100 / 54.  0.100
libavfilter     2. 77.100 /  2. 77.100
libswscale      2.  1.100 /  2.  1.100
libswresample   0. 15.100 /  0. 15.100
h264 @ 12eaac0] no frame!
    Last message repeated 1 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 23 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 74 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 64 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 34 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 49 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 24 times
[h264 @ 12eaac0] Partitioned H.264 support is incomplete
[h264 @ 12eaac0] no frame!
    Last message repeated 23 times
[h264 @ 12eaac0] sps_id out of range
[h264 @ 12eaac0] no frame!
    Last message repeated 148 times
[h264 @ 12eaac0] sps_id (32) out of range
    Last message repeated 1 times
[h264 @ 12eaac0] no frame!
    Last message repeated 33 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 128 times
[h264 @ 12eaac0] sps_id (32) out of range
    Last message repeated 1 times
[h264 @ 12eaac0] no frame!
    Last message repeated 3 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 3 times
[h264 @ 12eaac0] slice type too large (0) at 0 0
[h264 @ 12eaac0] decode_slice_header error
[h264 @ 12eaac0] no frame!
    Last message repeated 309 times
[h264 @ 12eaac0] sps_id (32) out of range
    Last message repeated 1 times
[h264 @ 12eaac0] no frame!
    Last message repeated 192 times
[h264 @ 12eaac0] Partitioned H.264 support is incomplete
[h264 @ 12eaac0] no frame!
    Last message repeated 73 times
[h264 @ 12eaac0] sps_id (32) out of range
    Last message repeated 1 times
[h264 @ 12eaac0] no frame!
    Last message repeated 99 times
[h264 @ 12eaac0] sps_id (32) out of range
    Last message repeated 1 times
[h264 @ 12eaac0] no frame!
    Last message repeated 197 times
[mov,mp4,m4a,3gp,3g2,mj2 @ 12e3100] decoding for stream 0 failed
[mov,mp4,m4a,3gp,3g2,mj2 @ 12e3100] Could not find codec parameters
(Video: h264 (avc1 / 0x31637661), 393539 kb/s)
out.mp4: could not find codec parameters

I really do not know where the issue is, except it has to be something to do with the way the streams are being set up. I've looked at bits of code from where other people are doing a similar thing, and tried to use this advice in setting up the streams, but to no avail!

---

The final code which gave me a H.264/AAC muxed (synced) file is as follows. First a bit of background information. The data is coming from an IP camera. The data is presented via a 3rd party API as video/audio packets. The video packets are presented as the RTP payload data (no header) and consist of NALU's that are reconstructed and converted to H.264 video in Annex B format. AAC audio is presented as raw AAC and is converted to adts format to enable playback. These packets have been put into a bitstream format that allows the transmission of the timestamp (64 bit milliseconds since Jan 1 1970) along with a few other things.

This is more or less a prototype and is not clean in any respects. It probably leaks bad. I do however, hope this helps anyone else out trying to achieve something similar to what I am.

Globals:

AVFormatContext* oc = NULL;
AVCodecContext* videoContext = NULL;
AVStream* videoStream = NULL;
AVCodecContext* audioContext = NULL;
AVStream* audioStream = NULL;
AVCodec* videoCodec = NULL;
AVCodec* audioCodec = NULL;
int vi = 0;  // Video stream
int ai = 1;  // Audio stream

uint64_t firstVideoTimeStamp = 0;
uint64_t firstAudioTimeStamp = 0;
int audioStartOffset = 0;

char* filename = NULL;

Boolean first = TRUE;

int videoFrameNumber = 0;
int audioFrameNumber = 0;

Main:

int main(int argc, char* argv[])
{
    if (argc != 3)
    {   
        cout << argv[0] << " <stream playback file> <output mp4 file>" << endl;
        return 0;
    }
    char* input_stream_file = argv[1];
    filename = argv[2];

    av_register_all();    

    fstream inFile;
    inFile.open(input_stream_file, ios::in);

    // Used to store the latest pps & sps frames
    unsigned char* ppsFrame = NULL;
    int ppsFrameLength = 0;
    unsigned char* spsFrame = NULL;
    int spsFrameLength = 0;

    // Setup MP4 output file
    AVOutputFormat* fmt = av_guess_format( 0, filename, 0 );
    oc = avformat_alloc_context();
    oc->oformat = fmt;
    strcpy(oc->filename, filename);

    // Setup the bitstream filter for AAC in adts format.  Could probably also achieve
    // this by stripping the first 7 bytes!
    AVBitStreamFilterContext* bsfc = av_bitstream_filter_init("aac_adtstoasc");
    if (!bsfc)
    {       
        cout << "Error creating adtstoasc filter" << endl;
        return -1;
    }

    while (inFile.good())
    {
        TcpAVDataBlock* block = new TcpAVDataBlock();
        block->readStruct(inFile);
        DateTime dt = block->getTimestampAsDateTime();
        switch (block->getPacketTyp

Answer 1

Please let me sum it up: the problem with your (original) code was that the input to av_interleaved_write_frame() should not start with the packet length. The file may still be playable if you don't strip the 00 00 00 01 start codes, but that IMHO is a resilience behavior of the player, and I would not count on this.