Ticket #560: transform_skip+transformshift_overflow.patch

source/Lib/TLibCommon/TComTrQuant.cpp

@@ -1173 +1173 @@
 #else
     UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-    UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;  // Represents scaling through forward transform
+    Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;  // Represents scaling through forward transform
 
     Int iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift;                // Right shift of non-RDOQ quantizer;  level = (coeff*uiQ + offset)>>q_bits
 
@@ -1252 +1252 @@
 #else
   UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-  UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
+  Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
 
   iShift = QUANT_IQUANT_SHIFT - QUANT_SHIFT - iTransformShift;
 
@@ -1636 +1636 @@
 #else
   UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-  UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
-  Int j,k;
-  for (j = 0; j < height; j++)
-  {    
-    for(k = 0; k < width; k ++)
-    {
-      psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << iTransformShift;      
+  Int  shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
+  UInt transformSkipShift;
+  Int  j,k;
+  if(shift >= 0)
+  {
+    transformSkipShift = shift;
+    for (j = 0; j < height; j++)
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << transformSkipShift;      
+      }
     }
   }
+  else
+  {
+    //The case when uiBitDepth > 13
+    UInt iAdd;
+    transformSkipShift = -shift;
+    iAdd = (1 << (transformSkipShift -1));
+    for (j = 0; j < height; j++)
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        psCoeff[j*height + k] = (piBlkResi[j * uiStride + k] + iAdd) >> transformSkipShift;      
+      }
+    }
+  }
 }
 
 /** Wrapper function between HM interface and core NxN transform skipping 
@@ -1662 +1681 @@
 #else
   UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-  UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
-  UInt iAdd = (1 << (iTransformShift -1));
-  Int j,k;
-  for ( j = 0; j < height; j++ )
-  {    
-    for(k = 0; k < width; k ++)
-    {
-      pResidual[j * uiStride + k] =  (plCoef[j*width+k] + iAdd) >> iTransformShift;
+  Int  shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
+  UInt transformSkipShift; 
+  Int  j,k;
+  if(shift > 0)
+  {
+    UInt iAdd;
+    transformSkipShift = shift;
+    iAdd = (1 << (transformSkipShift -1));
+    for ( j = 0; j < height; j++ )
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        pResidual[j * uiStride + k] =  (plCoef[j*width+k] + iAdd) >> transformSkipShift;
+      } 
     }
   }
+  else
+  {
+    //The case when uiBitDepth >= 13
+    transformSkipShift = - shift;
+    for ( j = 0; j < height; j++ )
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        pResidual[j * uiStride + k] =  plCoef[j*width+k] << transformSkipShift;
+      }
+    }
+  }
 }
 #endif
 

source/Lib/TLibDecoder/TDecSbac.cpp

@@ -1114 +1114 @@
     return;
   }
   
-  UInt useTansformSkip;
-  m_pcTDecBinIf->decodeBin( useTansformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) );
-  if(eTType!= TEXT_LUMA && uiDepth == 4)
+  UInt useTransformSkip;
+  m_pcTDecBinIf->decodeBin( useTransformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) );
+  if(eTType!= TEXT_LUMA)
   {
-    uiDepth --;
+    const UInt uiLog2TrafoSize = g_aucConvertToBit[pcCU->getSlice()->getSPS()->getMaxCUWidth()] + 2 - uiDepth;
+        if(uiLog2TrafoSize == 2) 
+        { 
+          uiDepth --;
+    }
   }
   DTRACE_CABAC_VL( g_nSymbolCounter++ )
   DTRACE_CABAC_T("\tparseTransformSkip()");
   DTRACE_CABAC_T( "\tsymbol=" )
-  DTRACE_CABAC_V( useTansformSkip )
+  DTRACE_CABAC_V( useTransformSkip )
   DTRACE_CABAC_T( "\tAddr=" )
   DTRACE_CABAC_V( pcCU->getAddr() )
   DTRACE_CABAC_T( "\tetype=" )
@@ -1132 +1136 @@
   DTRACE_CABAC_V( uiAbsPartIdx )
   DTRACE_CABAC_T( "\n" )
 
-  pcCU->setTransformSkipSubParts( useTansformSkip, eTType, uiAbsPartIdx, uiDepth);
+  pcCU->setTransformSkipSubParts( useTransformSkip, eTType, uiAbsPartIdx, uiDepth);
 }
 #endif