Ticket #596: 14-bit-fixes-HM71rc1_560_595_596_20120628.patch

source/Lib/TLibCommon/CommonDef.h

@@ -112 +112 @@
 
 #define MAX_UINT                    0xFFFFFFFFU ///< max. value of unsigned 32-bit integer
 #define MAX_INT                     2147483647  ///< max. value of signed 32-bit integer
+#define MAX_INT64                                       0x7FFFFFFFFFFFFFFF  ///< max. value of signed 64-bit integer
 #define MAX_DOUBLE                  1.7e+308    ///< max. value of double-type value
 
 #define MIN_QP                      0

source/Lib/TLibCommon/TComInterpolationFilter.cpp

@@ -124 +124 @@
   else
   {
     Int shift = IF_INTERNAL_PREC - ( g_uiBitDepth + g_uiBitIncrement );
-    Short offset = IF_INTERNAL_OFFS + (1 << (shift - 1));
+    Short offset = IF_INTERNAL_OFFS;
+    offset += shift?(1 << (shift - 1)):0;
     Short maxVal = g_uiIBDI_MAX;
     Short minVal = 0;
     for (row = 0; row < height; row++)

source/Lib/TLibCommon/TComTrQuant.cpp

@@ -1173 +1173 @@
 #else
     UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-    UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;  // Represents scaling through forward transform
+    Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;  // Represents scaling through forward transform
 
     Int iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift;                // Right shift of non-RDOQ quantizer;  level = (coeff*uiQ + offset)>>q_bits
 
@@ -1252 +1252 @@
 #else
   UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-  UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
+  Int iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
 
   iShift = QUANT_IQUANT_SHIFT - QUANT_SHIFT - iTransformShift;
 
@@ -1636 +1636 @@
 #else
   UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-  UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
-  Int j,k;
-  for (j = 0; j < height; j++)
-  {    
-    for(k = 0; k < width; k ++)
-    {
-      psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << iTransformShift;      
+  Int  shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize;
+  UInt transformSkipShift;
+  Int  j,k;
+  if(shift >= 0)
+  {
+    transformSkipShift = shift;
+    for (j = 0; j < height; j++)
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << transformSkipShift;      
+      }
     }
   }
+  else
+  {
+    //The case when uiBitDepth > 13
+    UInt iAdd;
+    transformSkipShift = -shift;
+    iAdd = (1 << (transformSkipShift -1));
+    for (j = 0; j < height; j++)
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        psCoeff[j*height + k] = (piBlkResi[j * uiStride + k] + iAdd) >> transformSkipShift;      
+      }
+    }
+  }
 }
 
 /** Wrapper function between HM interface and core NxN transform skipping 
@@ -1662 +1681 @@
 #else
   UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement;
 #endif
-  UInt iTransformShift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
-  UInt iAdd = (1 << (iTransformShift -1));
-  Int j,k;
-  for ( j = 0; j < height; j++ )
-  {    
-    for(k = 0; k < width; k ++)
-    {
-      pResidual[j * uiStride + k] =  (plCoef[j*width+k] + iAdd) >> iTransformShift;
+  Int  shift = MAX_TR_DYNAMIC_RANGE - uiBitDepth - uiLog2TrSize; 
+  UInt transformSkipShift; 
+  Int  j,k;
+  if(shift > 0)
+  {
+    UInt iAdd;
+    transformSkipShift = shift;
+    iAdd = (1 << (transformSkipShift -1));
+    for ( j = 0; j < height; j++ )
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        pResidual[j * uiStride + k] =  (plCoef[j*width+k] + iAdd) >> transformSkipShift;
+      } 
     }
   }
+  else
+  {
+    //The case when uiBitDepth >= 13
+    transformSkipShift = - shift;
+    for ( j = 0; j < height; j++ )
+    {    
+      for(k = 0; k < width; k ++)
+      {
+        pResidual[j * uiStride + k] =  plCoef[j*width+k] << transformSkipShift;
+      }
+    }
+  }
 }
 #endif
 
@@ -2127 +2164 @@
 
   if( pcCU->getSlice()->getPPS()->getSignHideFlag() && uiAbsSum>=2)
   {
-    Int rdFactor = (Int)((Double)(g_invQuantScales[m_cQP.rem()]*g_invQuantScales[m_cQP.rem()]<<(2*m_cQP.m_iPer))/m_dLambda/16 + 0.5) ;
+    Int64 rdFactor = (Int64)((Double)((Int64)g_invQuantScales[m_cQP.rem()]*(Int64)g_invQuantScales[m_cQP.rem()]<<(2*(Int64)m_cQP.m_iPer))/m_dLambda/16 + 0.5) ;
 #if !FIXED_SBH_THRESHOLD
     Int tsig = pcCU->getSlice()->getPPS()->getTSIG() ;
 #endif
@@ -2176 +2213 @@
         if( signbit!=(absSum&0x1) )  // hide but need tune
         {
           // calculate the cost 
-          Int minCostInc = MAX_INT,  minPos =-1, finalChange=0, curCost=MAX_INT, curChange=0;
+          Int64 minCostInc = MAX_INT64, curCost=MAX_INT64;
+          Int minPos =-1, finalChange=0, curChange=0;
 
           for( n = (lastCG==1?lastNZPosInCG:SCAN_SET_SIZE-1) ; n >= 0; --n )
           {
             UInt uiBlkPos   = scan[ n + subPos ];
             if(piDstCoeff[ uiBlkPos ] != 0 )
             {
-              Int costUp   = rdFactor * ( - deltaU[uiBlkPos] ) + rateIncUp[uiBlkPos] ;
-              Int costDown = rdFactor * (   deltaU[uiBlkPos] ) + rateIncDown[uiBlkPos] 
+              Int64 costUp   = rdFactor * ( (Int64)- deltaU[uiBlkPos] ) + rateIncUp[uiBlkPos] ;
+              Int64 costDown = rdFactor * ( (Int64) deltaU[uiBlkPos] ) + rateIncDown[uiBlkPos] 
                 -   ( abs(piDstCoeff[uiBlkPos])==1?((1<<15)+sigRateDelta[uiBlkPos]):0 );
 
               if(lastCG==1 && lastNZPosInCG==n && abs(piDstCoeff[uiBlkPos])==1)
@@ -2202 +2240 @@
                 curChange = -1 ;
                 if(n==firstNZPosInCG && abs(piDstCoeff[uiBlkPos])==1)
                 {
-                  curCost = MAX_INT ;
+                  curCost = MAX_INT64 ;
                 }
                 else
                 {
@@ -2220 +2258 @@
                 UInt thissignbit = (plSrcCoeff[uiBlkPos]>=0?0:1);
                 if(thissignbit != signbit )
                 {
-                  curCost = MAX_INT;
+                  curCost = MAX_INT64;
                 }
               }
             }

source/Lib/TLibCommon/TComWeightPrediction.cpp

@@ -79 +79 @@
   Int offset  = wp0[0].offset;
   Int shiftNum = IF_INTERNAL_PREC - ( g_uiBitDepth + g_uiBitIncrement );
   Int shift   = wp0[0].shift + shiftNum;
-  Int round   = (1<<(shift-1)) * bRound;
+  Int round   = shift?(1<<(shift-1)) * bRound:0;
   Int w1      = wp1[0].w;
 
   UInt  iSrc0Stride = pcYuvSrc0->getStride();
@@ -105 +105 @@
   w0      = wp0[1].w;
   offset  = wp0[1].offset;
   shift   = wp0[1].shift + shiftNum;
-  round   = (1<<(shift-1));
+  round   = shift?(1<<(shift-1)):0;
   w1      = wp1[1].w;
 
   iSrc0Stride = pcYuvSrc0->getCStride();
@@ -132 +132 @@
   w0      = wp0[2].w;
   offset  = wp0[2].offset;
   shift   = wp0[2].shift + shiftNum;
-  round   = (1<<(shift-1));
+  round   = shift?(1<<(shift-1)):0;
   w1      = wp1[2].w;
 
   for ( y = iHeight-1; y >= 0; y-- )
@@ -175 +175 @@
   Int offset  = wp0[0].offset;
   Int shiftNum = IF_INTERNAL_PREC - ( g_uiBitDepth + g_uiBitIncrement );
   Int shift   = wp0[0].shift + shiftNum;
-  Int round   = (1<<(shift-1));
+  Int round   = shift?(1<<(shift-1)):0;
   UInt  iSrc0Stride = pcYuvSrc0->getStride();
   UInt  iDstStride  = rpcYuvDst->getStride();
   
@@ -197 +197 @@
   w0      = wp0[1].w;
   offset  = wp0[1].offset;
   shift   = wp0[1].shift + shiftNum;
-  round   = (1<<(shift-1));
+  round   = shift?(1<<(shift-1)):0;
 
   iSrc0Stride = pcYuvSrc0->getCStride();
   iDstStride  = rpcYuvDst->getCStride();
@@ -221 +221 @@
   w0      = wp0[2].w;
   offset  = wp0[2].offset;
   shift   = wp0[2].shift + shiftNum;
-  round   = (1<<(shift-1));
+  round   = shift?(1<<(shift-1)):0;
 
   for ( y = iHeight-1; y >= 0; y-- )
   {

source/Lib/TLibDecoder/TDecSbac.cpp

@@ -1114 +1114 @@
     return;
   }
   
-  UInt useTansformSkip;
-  m_pcTDecBinIf->decodeBin( useTansformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) );
-  if(eTType!= TEXT_LUMA && uiDepth == 4)
+  UInt useTransformSkip;
+  m_pcTDecBinIf->decodeBin( useTransformSkip , m_cTransformSkipSCModel.get( 0, eTType? TEXT_CHROMA: TEXT_LUMA, 0 ) );
+  if(eTType!= TEXT_LUMA)
   {
-    uiDepth --;
+    const UInt uiLog2TrafoSize = g_aucConvertToBit[pcCU->getSlice()->getSPS()->getMaxCUWidth()] + 2 - uiDepth;
+        if(uiLog2TrafoSize == 2) 
+        { 
+          uiDepth --;
+    }
   }
   DTRACE_CABAC_VL( g_nSymbolCounter++ )
   DTRACE_CABAC_T("\tparseTransformSkip()");
   DTRACE_CABAC_T( "\tsymbol=" )
-  DTRACE_CABAC_V( useTansformSkip )
+  DTRACE_CABAC_V( useTransformSkip )
   DTRACE_CABAC_T( "\tAddr=" )
   DTRACE_CABAC_V( pcCU->getAddr() )
   DTRACE_CABAC_T( "\tetype=" )
@@ -1132 +1136 @@
   DTRACE_CABAC_V( uiAbsPartIdx )
   DTRACE_CABAC_T( "\n" )
 
-  pcCU->setTransformSkipSubParts( useTansformSkip, eTType, uiAbsPartIdx, uiDepth);
+  pcCU->setTransformSkipSubParts( useTransformSkip, eTType, uiAbsPartIdx, uiDepth);
 }
 #endif