Ticket #596: 14-bit-overflow_onTopOfHM71rc1.patch

source/Lib/TLibCommon/CommonDef.h

@@ -112 +112 @@
 
 #define MAX_UINT                    0xFFFFFFFFU ///< max. value of unsigned 32-bit integer
 #define MAX_INT                     2147483647  ///< max. value of signed 32-bit integer
+#define MAX_INT64                                       0x7FFFFFFFFFFFFFFF  ///< max. value of signed 64-bit integer
 #define MAX_DOUBLE                  1.7e+308    ///< max. value of double-type value
 
 #define MIN_QP                      0

source/Lib/TLibCommon/TComTrQuant.cpp

@@ -2127 +2127 @@
 
   if( pcCU->getSlice()->getPPS()->getSignHideFlag() && uiAbsSum>=2)
   {
-    Int rdFactor = (Int)((Double)(g_invQuantScales[m_cQP.rem()]*g_invQuantScales[m_cQP.rem()]<<(2*m_cQP.m_iPer))/m_dLambda/16 + 0.5) ;
+    Int64 rdFactor = (Int64)((Double)((Int64)g_invQuantScales[m_cQP.rem()]*(Int64)g_invQuantScales[m_cQP.rem()]<<(2*(Int64)m_cQP.m_iPer))/m_dLambda/16 + 0.5) ;
 #if !FIXED_SBH_THRESHOLD
     Int tsig = pcCU->getSlice()->getPPS()->getTSIG() ;
 #endif
@@ -2176 +2176 @@
         if( signbit!=(absSum&0x1) )  // hide but need tune
         {
           // calculate the cost 
-          Int minCostInc = MAX_INT,  minPos =-1, finalChange=0, curCost=MAX_INT, curChange=0;
+          Int64 minCostInc = MAX_INT64, curCost=MAX_INT64;
+          Int minPos =-1, finalChange=0, curChange=0;
 
           for( n = (lastCG==1?lastNZPosInCG:SCAN_SET_SIZE-1) ; n >= 0; --n )
           {
             UInt uiBlkPos   = scan[ n + subPos ];
             if(piDstCoeff[ uiBlkPos ] != 0 )
             {
-              Int costUp   = rdFactor * ( - deltaU[uiBlkPos] ) + rateIncUp[uiBlkPos] ;
-              Int costDown = rdFactor * (   deltaU[uiBlkPos] ) + rateIncDown[uiBlkPos] 
+              Int64 costUp   = rdFactor * ( (Int64)- deltaU[uiBlkPos] ) + rateIncUp[uiBlkPos] ;
+              Int64 costDown = rdFactor * ( (Int64) deltaU[uiBlkPos] ) + rateIncDown[uiBlkPos] 
                 -   ( abs(piDstCoeff[uiBlkPos])==1?((1<<15)+sigRateDelta[uiBlkPos]):0 );
 
               if(lastCG==1 && lastNZPosInCG==n && abs(piDstCoeff[uiBlkPos])==1)
@@ -2202 +2203 @@
                 curChange = -1 ;
                 if(n==firstNZPosInCG && abs(piDstCoeff[uiBlkPos])==1)
                 {
-                  curCost = MAX_INT ;
+                  curCost = MAX_INT64 ;
                 }
                 else
                 {
@@ -2220 +2221 @@
                 UInt thissignbit = (plSrcCoeff[uiBlkPos]>=0?0:1);
                 if(thissignbit != signbit )
                 {
-                  curCost = MAX_INT;
+                  curCost = MAX_INT64;
                 }
               }
             }