Ticket #597: rdFactorFix.patch

File rdFactorFix.patch, 675 bytes (added by wangj, 10 years ago)
  • ./source/Lib/TLibCommon/TComTrQuant.cpp

     
    21142114
    21152115  if( pcCU->getSlice()->getPPS()->getSignHideFlag() && uiAbsSum>=2)
    21162116  {
    2117     Int rdFactor = (Int)((Double)(g_invQuantScales[m_cQP.rem()]*g_invQuantScales[m_cQP.rem()]<<(2*m_cQP.m_iPer))/m_dLambda/16 + 0.5) ;
     2117    Int rdFactor = (Int)((Double)(g_invQuantScales[m_cQP.rem()])*(Double)(g_invQuantScales[m_cQP.rem()])*(Double)(1<<(2*m_cQP.m_iPer))/m_dLambda/16/(Double)(1<<(2*g_uiBitIncrement)) + 0.5);
    21182118#if !FIXED_SBH_THRESHOLD
    21192119    Int tsig = pcCU->getSlice()->getPPS()->getTSIG() ;
    21202120#endif