Ticket #946: dequant_overflow.patch

File dequant_overflow.patch, 613 bytes (added by bheng, 11 years ago)

  • source/Lib/TLibCommon/TComTrQuant.cpp

     
    11361136  iShift = QUANT_IQUANT_SHIFT - QUANT_SHIFT - iTransformShift;
    11371137
    11381138  TCoeff clipQCoef;
    1139   const Int bitRange = min( 15, ( Int )( 12 + uiLog2TrSize + bitDepth - m_cQP.m_iPer) );
     1139  const Int bitRange = min( 15, ( Int )( 12 + uiLog2TrSize + bitDepth - m_cQP.m_iPer - (m_cQP.m_iRem > 3)) );
    11401140  const Int levelLimit = 1 << bitRange;
    11411141
    11421142  if(getUseScalingList())