Microsoft Software H264 Encoder Produces Progressively Degrading Quality

Question

I have a functioning encoder using Media Foundation, which uses hardware acceleration. I have decided to add Microsoft's software H264 encoder as a fallback if hardware acceleration is for some reason not available.

The problem I have, is when I begin streaming from the encoder, the quality starts off fine and then rapidly degrades until nothing is readable/viewable at all. This seems to be triggered the moment there is any motion. This problems only exists when I use the software encoder. When I am using any of the hardware encoders, it may degrade for a moment when there is movement, but it quickly restores the quality somehow.

Here is a frame before any sort of movement:

Here is how it looks after any movement occurred:

Here is my encoder configuration

HRESULT configure_encoder(const CComPtr<IMFTransform>& in_transform, CComPtr<IMFDXGIDeviceManager>& in_device_manager,
    const DWORD in_input_stream_id, const DWORD output_stream_id
)
{
    HRESULT hr = S_OK;

    // Sets or clears the Direct3D Device Manager for DirectX Video Acceleration (DXVA)
    if (FAILED(hr = in_transform->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, reinterpret_cast<ULONG_PTR>(in_device_manager.p))))
    {
        std::cout << "Failed to enable DXVA acceleration for encoder";
        //return hr;
    }

    // Create output type
    CComPtr<IMFMediaType> output_type;
    if (FAILED(hr = MFCreateMediaType(&output_type)))
        return hr;

    // Set output type
    if (FAILED(hr = output_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video)))
        return hr;
    if (FAILED(hr = output_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264)))
        return hr;
    if (FAILED(hr = output_type->SetUINT32(MF_MT_AVG_BITRATE, encode_bitrate)))
        return hr;
    if (FAILED(hr = output_type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High)))
        return hr;
    if (FAILED(hr = MFSetAttributeSize(output_type, MF_MT_FRAME_SIZE, encode_width, encode_height)))
        return hr;
    if (FAILED(hr = MFSetAttributeRatio(output_type, MF_MT_FRAME_RATE, 60, 1)))
        return hr;
    if (FAILED(hr = output_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlaceMode::MFVideoInterlace_Progressive)))
        return hr;

    // Set output type
    if (FAILED(hr = in_transform->SetOutputType(output_stream_id, output_type, 0)))
        return hr;

    // Input type
    CComPtr<IMFMediaType> input_type;

    if (FAILED(hr = in_transform->GetInputAvailableType(in_input_stream_id, 0, &input_type)))
        return hr;

    // Input type settings
    if (FAILED(hr = input_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video)))
        return hr;
    if (FAILED(hr = input_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12)))
        return hr;
    if (FAILED(hr = MFSetAttributeSize(input_type, MF_MT_FRAME_SIZE, encode_width, encode_height)))
        return hr;
    if (FAILED(hr = MFSetAttributeRatio(input_type, MF_MT_FRAME_RATE, 60, 1)))
        return hr;
    if (FAILED(hr = input_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlaceMode::MFVideoInterlace_Progressive)))
        return hr;

    // Set input type
    if (FAILED(hr = in_transform->SetInputType(in_input_stream_id, input_type, 0)))
        return hr;

    std::cout << "- Set encoder configuration" << std::endl;

    // AMD encoder crashes on this line
    /*DWORD flags;
    if (FAILED(hr = inTransform->GetInputStatus(0, &flags)))
        return hr;*/

    return hr;
}

I tried tampering with the following codec API attributes hoping something would change, I wasn't able to make any noticeable difference.

// Get Codec API pointer
ICodecAPI* mpCodecAPI = NULL;
if (FAILED(hr = encoder_transform->QueryInterface(IID_PPV_ARGS(&mpCodecAPI))))
return hr;

VARIANT var;

// Set mode to CBR
var = { 0 };
var.vt = VT_UI4;
var.lVal = eAVEncCommonRateControlMode_CBR;
if (FAILED(hr = mpCodecAPI->SetValue(&CODECAPI_AVEncCommonRateControlMode, &var)))
return hr;

var.vt = VT_BOOL;
var.boolVal = VARIANT_TRUE;
if (FAILED(hr = mpCodecAPI->SetValue(&CODECAPI_AVLowLatencyMode, &var)))
return hr;

var = { 0 };
var.vt = VT_UI4;
var.lVal = 100;
if (FAILED(hr = mpCodecAPI->SetValue(&CODECAPI_AVEncCommonQuality, &var)))
return hr;

var = { 0 };
var.vt = VT_UI4;
var.lVal = 4;
if (FAILED(hr = mpCodecAPI->SetValue(&CODECAPI_AVEncVideoMaxNumRefFrame, &var)))
return hr;

var = { 0 };
var.vt = VT_UI4;
var.lVal = 0;
if (FAILED(hr = mpCodecAPI->SetValue(&CODECAPI_AVEncMPVDefaultBPictureCount, &var)))
return hr;

var = { 0 };
var.vt = VT_UI4;
var.lVal = 0;
if (FAILED(hr = mpCodecAPI->SetValue(&CODECAPI_AVEncMPVGOPSize, &var)))
return hr;```

Answer 1

The software encoder isn't set in your code.

    CComPtr<IMFAttributes> attrs;
    MFCreateAttributes(&attrs, 0);
    attrs->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, false);
    hr = MFCreateSinkWriterFromURL(file, NULL, attrs, &pSinkWriter);