Ticket #560: transform_skip+transformshift_overflow.patch

File transform_skip+transformshift_overflow.patch, 4.6 KB (added by pandrivon, 10 years ago)

transform skip patch + overflow transformShift fix + typo fix

  • source/Lib/TLibCommon/TComTrQuant.cpp

     
    11731173#else
    11741174    UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
    11751175#endif
    1176     UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;  // Represents scaling through forward transform
     1176    Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;  // Represents scaling through forward transform
    11771177
    11781178    Int iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift;                // Right shift of non-RDOQ quantizer;  level = (coeff*uiQ + offset)>>q_bits
    11791179
     
    12521252#else
    12531253  UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
    12541254#endif
    1255   UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
     1255  Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
    12561256
    12571257  iShift = QUANT_IQUANT_SHIFT - QUANT_SHIFT - iTransformShift;
    12581258
     
    16361636#else
    16371637  UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
    16381638#endif
    1639   UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
    1640   Int j,k;
    1641   for (j = 0; j < height; j++)
    1642   {   
    1643     for(k = 0; k < width; k ++)
    1644     {
    1645       psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << iTransformShift;     
     1639  Int  shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
     1640  UInt transformSkipShift;
     1641  Int  j,k;
     1642  if(shift >= 0)
     1643  {
     1644    transformSkipShift = shift;
     1645    for (j = 0; j < height; j++)
     1646    {   
     1647      for(k = 0; k < width; k ++)
     1648      {
     1649        psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << transformSkipShift;     
     1650      }
    16461651    }
    16471652  }
     1653  else
     1654  {
     1655    //The case when uiBitDepth > 13
     1656    UInt iAdd;
     1657    transformSkipShift = -shift;
     1658    iAdd = (1 << (transformSkipShift -1));
     1659    for (j = 0; j < height; j++)
     1660    {   
     1661      for(k = 0; k < width; k ++)
     1662      {
     1663        psCoeff[j*height + k] = (piBlkResi[j * uiStride + k] + iAdd) >> transformSkipShift;     
     1664      }
     1665    }
     1666  }
    16481667}
    16491668
    16501669/** Wrapper function between HM interface and core NxN transform skipping
     
    16621681#else
    16631682  UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
    16641683#endif
    1665   UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
    1666   UInt iAdd = (1 << (iTransformShift -1));
    1667   Int j,k;
    1668   for ( j = 0; j < height; j++ )
    1669   {   
    1670     for(k = 0; k < width; k ++)
    1671     {
    1672       pResidual[j * uiStride + k] =  (plCoef[j*width+k] + iAdd) >> iTransformShift;
     1684  Int  shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
     1685  UInt transformSkipShift;
     1686  Int  j,k;
     1687  if(shift > 0)
     1688  {
     1689    UInt iAdd;
     1690    transformSkipShift = shift;
     1691    iAdd = (1 << (transformSkipShift -1));
     1692    for ( j = 0; j < height; j++ )
     1693    {   
     1694      for(k = 0; k < width; k ++)
     1695      {
     1696        pResidual[j * uiStride + k] =  (plCoef[j*width+k] + iAdd) >> transformSkipShift;
     1697      }
    16731698    }
    16741699  }
     1700  else
     1701  {
     1702    //The case when uiBitDepth >= 13
     1703    transformSkipShift = - shift;
     1704    for ( j = 0; j < height; j++ )
     1705    {   
     1706      for(k = 0; k < width; k ++)
     1707      {
     1708        pResidual[j * uiStride + k] =  plCoef[j*width+k] << transformSkipShift;
     1709      }
     1710    }
     1711  }
    16751712}
    16761713#endif
    16771714
  • source/Lib/TLibDecoder/TDecSbac.cpp

     
    11141114    return;
    11151115  }
    11161116 
    1117   UInt useTansformSkip;
    1118   m_pcTDecBinIf->decodeBin( useTansformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) );
    1119   if(eTType!= TEXT_LUMA && uiDepth == 4)
     1117  UInt useTransformSkip;
     1118  m_pcTDecBinIf->decodeBin( useTransformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) );
     1119  if(eTType!= TEXT_LUMA)
    11201120  {
    1121     uiDepth --;
     1121    const UInt uiLog2TrafoSize = g_aucConvertToBit[pcCU->getSlice()->getSPS()->getMaxCUWidth()] + 2 - uiDepth;
     1122        if(uiLog2TrafoSize == 2)
     1123        {
     1124          uiDepth --;
     1125    }
    11221126  }
    11231127  DTRACE_CABAC_VL( g_nSymbolCounter++ )
    11241128  DTRACE_CABAC_T("\tparseTransformSkip()");
    11251129  DTRACE_CABAC_T( "\tsymbol=" )
    1126   DTRACE_CABAC_V( useTansformSkip )
     1130  DTRACE_CABAC_V( useTransformSkip )
    11271131  DTRACE_CABAC_T( "\tAddr=" )
    11281132  DTRACE_CABAC_V( pcCU->getAddr() )
    11291133  DTRACE_CABAC_T( "\tetype=" )
     
    11321136  DTRACE_CABAC_V( uiAbsPartIdx )
    11331137  DTRACE_CABAC_T( "\n" )
    11341138
    1135   pcCU->setTransformSkipSubParts( useTansformSkip, eTType, uiAbsPartIdx, uiDepth);
     1139  pcCU->setTransformSkipSubParts( useTransformSkip, eTType, uiAbsPartIdx, uiDepth);
    11361140}
    11371141#endif
    11381142