[XviD-devel] bug in xvid decoder when parsing input bitstream

[liang jian]

Hello every one, I have found a bug in xvid decoder when parsing input
bitstream, this bug can be reproduced through the following steps:

1) prepare a 16 x 16 image with YUY2 video format, clear it with black
color(y = 16, cb = cr = 128)

2) encode this image into an I-frame using xvid(simple profile level 3), the
coded stream is 52 bytes long.

3) allocate one page using VirtualAlloc (use mmap in linux), copy the
encoded stream to the end of this page.

4) decode the encoded stream in that page using xvid, an read access
violation occurs in BitstreamSkip() function when it execute the following
line:
    tmp = *((uint32_t *) bs->tail + 2);

BitstreamSkip() function update bit position in the bitstream, and always
read a uint32_t value which is next to the uint32_t value contains the
current bit position, when decoder reach the end of the stream,
BitstreamSkip() will unavoidably read the memory which address is bigger
than the last byte of the input stream(and in the above case this addreass
is an invalid address).


the code is attached.
-------------- next part --------------
#include "stdafx.h"
#include <xvid.h>

static void create_codec(void **phEncoder, void **phDecoder,
                         int nWidth, int nHeight)
{
    int nErr;

    // initialize xvid library
    xvid_gbl_init_t xvid_init;

    memset(&xvid_init, 0, sizeof(xvid_gbl_init_t));
    xvid_init.version = XVID_VERSION;
    xvid_init.cpu_flags = 0;    // auto detect

    nErr = xvid_global(NULL, XVID_GBL_INIT, &xvid_init, NULL);
    ASSERT(nErr >= 0);

    // create encoder
    xvid_enc_create_t enc_create;

    memset(&enc_create, 0, sizeof(xvid_enc_create_t));
    enc_create.version = XVID_VERSION;
    enc_create.profile = XVID_PROFILE_S_L3;
    enc_create.width = nWidth;
    enc_create.height = nHeight;
    enc_create.fincr = 1;
    enc_create.fbase = 25;
    enc_create.num_plugins = 0;

    nErr = xvid_encore(NULL, XVID_ENC_CREATE, &enc_create, NULL);
    ASSERT(nErr >= 0);

    *phEncoder = enc_create.handle;

    // create decoder
    xvid_dec_create_t dec_create;

    memset(&dec_create, 0, sizeof(xvid_dec_create_t));
    dec_create.version = XVID_VERSION;
    dec_create.width = nWidth;
    dec_create.height = nHeight;

    nErr = xvid_decore(NULL, XVID_DEC_CREATE, &dec_create, NULL);
    ASSERT(nErr >= 0);

    *phDecoder = dec_create.handle;
}

static int encode_image(void *hEncoder, void *pImage, int nPitch,
                        void *pFrame, int nMaxFrameLength)
{
    xvid_enc_frame_t xvid_enc_frame;
    xvid_enc_stats_t xvid_enc_stats;
    int nBytes;

    memset(&xvid_enc_frame, 0, sizeof(xvid_enc_frame_t));
    xvid_enc_frame.version = XVID_VERSION;
    xvid_enc_frame.vol_flags = 0;
    xvid_enc_frame.vop_flags = 0;
    xvid_enc_frame.type = XVID_TYPE_IVOP;
    xvid_enc_frame.bitstream = pFrame;
    xvid_enc_frame.length = nMaxFrameLength;
    xvid_enc_frame.input.csp = XVID_CSP_YUY2;
    xvid_enc_frame.input.plane[0] = pImage;
    xvid_enc_frame.input.stride[0] = nPitch;

    memset(&xvid_enc_stats, 0, sizeof(xvid_enc_stats_t));
    xvid_enc_stats.version = XVID_VERSION;

    nBytes = xvid_encore(hEncoder, XVID_ENC_ENCODE, &xvid_enc_frame, &xvid_enc_stats);
    ASSERT(nBytes > 0);

    return nBytes;
}

static void decode_image(void *hDecoder, void *pFrame, int nFrameSize,
                         void *pImage, int nPitch)
{
    uint8_t *pui8Stream, *pui8StreamStart;
    int nAllocSize;

    nAllocSize = (nFrameSize + 4095) & (~4095) ;
    pui8Stream = (uint8_t *)::VirtualAlloc(NULL, nAllocSize, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
    ASSERT(pui8Stream != NULL);

    pui8StreamStart = pui8Stream + nAllocSize - nFrameSize;
    memcpy(pui8StreamStart, pFrame, nFrameSize);

    // decode
    xvid_dec_frame_t xvid_dec_frame;
    int nBytesConsumed;

    memset(&xvid_dec_frame, 0, sizeof(xvid_dec_frame_t));
    xvid_dec_frame.version = XVID_VERSION;
    xvid_dec_frame.general = XVID_DISCONTINUITY;
    xvid_dec_frame.bitstream = pui8StreamStart;
    xvid_dec_frame.length = nFrameSize;
    xvid_dec_frame.output.csp = XVID_CSP_YUY2;
    xvid_dec_frame.output.plane[0] = pImage;
    xvid_dec_frame.output.stride[0] = nPitch;

    nBytesConsumed = xvid_decore(hDecoder, XVID_DEC_DECODE, &xvid_dec_frame, NULL);
    ASSERT(nBytesConsumed == nFrameSize);

    ::VirtualFree(pui8Stream, 0, MEM_RELEASE);
}

#define IMAGE_WIDTH     16
#define IMAGE_HEIGHT    16

int main()
{
    uint8_t aImage[IMAGE_WIDTH * 2 * IMAGE_HEIGHT];
    uint8_t aFrame[4096];
    void *hEncoder, *hDecoder;
    uint32_t *pui32Buf;
    int i, j, nSize;

    // clear image
    pui32Buf = (uint32_t *)aImage;
    for (i = 0; i < IMAGE_HEIGHT; i++) {
        for (j = 0; j < IMAGE_WIDTH / 2; j++)
            *(pui32Buf++) = 0x80108010;
    }

    create_codec(&hEncoder, &hDecoder, IMAGE_WIDTH, IMAGE_HEIGHT);

    // encode this image
    nSize = encode_image(hEncoder, aImage, IMAGE_WIDTH * 2, aFrame, 4096);

    memset(aImage, 0, sizeof(aImage));

    decode_image(hDecoder, aFrame, nSize, aImage, IMAGE_WIDTH * 2);

    xvid_encore(hEncoder, XVID_ENC_DESTROY, NULL, NULL);
    xvid_decore(hDecoder, XVID_DEC_DESTROY, NULL, NULL);

    return 0;
}