Ticket #560: transform_skip+transformshift_overflow.patch
File transform_skip+transformshift_overflow.patch, 4.6 KB (added by pandrivon, 12 years ago) |
---|
-
source/Lib/TLibCommon/TComTrQuant.cpp
1173 1173 #else 1174 1174 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 1175 1175 #endif 1176 UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; // Represents scaling through forward transform1176 Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; // Represents scaling through forward transform 1177 1177 1178 1178 Int iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift; // Right shift of non-RDOQ quantizer; level = (coeff*uiQ + offset)>>q_bits 1179 1179 … … 1252 1252 #else 1253 1253 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 1254 1254 #endif 1255 UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;1255 Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 1256 1256 1257 1257 iShift = QUANT_IQUANT_SHIFT - QUANT_SHIFT - iTransformShift; 1258 1258 … … 1636 1636 #else 1637 1637 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 1638 1638 #endif 1639 UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 1640 Int j,k; 1641 for (j = 0; j < height; j++) 1642 { 1643 for(k = 0; k < width; k ++) 1644 { 1645 psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << iTransformShift; 1639 Int shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 1640 UInt transformSkipShift; 1641 Int j,k; 1642 if(shift >= 0) 1643 { 1644 transformSkipShift = shift; 1645 for (j = 0; j < height; j++) 1646 { 1647 for(k = 0; k < width; k ++) 1648 { 1649 psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << transformSkipShift; 1650 } 1646 1651 } 1647 1652 } 1653 else 1654 { 1655 //The case when uiBitDepth > 13 1656 UInt iAdd; 1657 transformSkipShift = -shift; 1658 iAdd = (1 << (transformSkipShift -1)); 1659 for (j = 0; j < height; j++) 1660 { 1661 for(k = 0; k < width; k ++) 1662 { 1663 psCoeff[j*height + k] = (piBlkResi[j * uiStride + k] + iAdd) >> transformSkipShift; 1664 } 1665 } 1666 } 1648 1667 } 1649 1668 1650 1669 /** Wrapper function between HM interface and core NxN transform skipping … … 1662 1681 #else 1663 1682 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 1664 1683 #endif 1665 UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 1666 UInt iAdd = (1 << (iTransformShift -1)); 1667 Int j,k; 1668 for ( j = 0; j < height; j++ ) 1669 { 1670 for(k = 0; k < width; k ++) 1671 { 1672 pResidual[j * uiStride + k] = (plCoef[j*width+k] + iAdd) >> iTransformShift; 1684 Int shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 1685 UInt transformSkipShift; 1686 Int j,k; 1687 if(shift > 0) 1688 { 1689 UInt iAdd; 1690 transformSkipShift = shift; 1691 iAdd = (1 << (transformSkipShift -1)); 1692 for ( j = 0; j < height; j++ ) 1693 { 1694 for(k = 0; k < width; k ++) 1695 { 1696 pResidual[j * uiStride + k] = (plCoef[j*width+k] + iAdd) >> transformSkipShift; 1697 } 1673 1698 } 1674 1699 } 1700 else 1701 { 1702 //The case when uiBitDepth >= 13 1703 transformSkipShift = - shift; 1704 for ( j = 0; j < height; j++ ) 1705 { 1706 for(k = 0; k < width; k ++) 1707 { 1708 pResidual[j * uiStride + k] = plCoef[j*width+k] << transformSkipShift; 1709 } 1710 } 1711 } 1675 1712 } 1676 1713 #endif 1677 1714 -
source/Lib/TLibDecoder/TDecSbac.cpp
1114 1114 return; 1115 1115 } 1116 1116 1117 UInt useT ansformSkip;1118 m_pcTDecBinIf->decodeBin( useT ansformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) );1119 if(eTType!= TEXT_LUMA && uiDepth == 4)1117 UInt useTransformSkip; 1118 m_pcTDecBinIf->decodeBin( useTransformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) ); 1119 if(eTType!= TEXT_LUMA) 1120 1120 { 1121 uiDepth --; 1121 const UInt uiLog2TrafoSize = g_aucConvertToBit[pcCU->getSlice()->getSPS()->getMaxCUWidth()] + 2 - uiDepth; 1122 if(uiLog2TrafoSize == 2) 1123 { 1124 uiDepth --; 1125 } 1122 1126 } 1123 1127 DTRACE_CABAC_VL( g_nSymbolCounter++ ) 1124 1128 DTRACE_CABAC_T("\tparseTransformSkip()"); 1125 1129 DTRACE_CABAC_T( "\tsymbol=" ) 1126 DTRACE_CABAC_V( useT ansformSkip )1130 DTRACE_CABAC_V( useTransformSkip ) 1127 1131 DTRACE_CABAC_T( "\tAddr=" ) 1128 1132 DTRACE_CABAC_V( pcCU->getAddr() ) 1129 1133 DTRACE_CABAC_T( "\tetype=" ) … … 1132 1136 DTRACE_CABAC_V( uiAbsPartIdx ) 1133 1137 DTRACE_CABAC_T( "\n" ) 1134 1138 1135 pcCU->setTransformSkipSubParts( useT ansformSkip, eTType, uiAbsPartIdx, uiDepth);1139 pcCU->setTransformSkipSubParts( useTransformSkip, eTType, uiAbsPartIdx, uiDepth); 1136 1140 } 1137 1141 #endif 1138 1142