Index: /branches/SHM-dev/source/App/TAppEncoder/TAppEncCfg.cpp
===================================================================
--- /branches/SHM-dev/source/App/TAppEncoder/TAppEncCfg.cpp	(revision 876)
+++ /branches/SHM-dev/source/App/TAppEncoder/TAppEncCfg.cpp	(revision 877)
@@ -1145,4 +1145,7 @@
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
   ("CGSAdaptC",     m_nCGSAdaptiveChroma , 1, "adaptive chroma partition (only for the case of two chroma partitions)")
+#endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  ("CGSSizeRDO",     m_nCGSLutSizeRDO , 0, "RDOpt selection of best table size (effective when large maximum table size such as 8x8x8 is used)")
 #endif
 #endif
@@ -3767,4 +3770,8 @@
   printf("CGSAdaptC:%d " , m_nCGSAdaptiveChroma );
 #endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  printf("CGSSizeRDO:%d " , m_nCGSLutSizeRDO );
+#endif
+
   printf("\n\n");
   
Index: /branches/SHM-dev/source/App/TAppEncoder/TAppEncCfg.h
===================================================================
--- /branches/SHM-dev/source/App/TAppEncoder/TAppEncCfg.h	(revision 876)
+++ /branches/SHM-dev/source/App/TAppEncoder/TAppEncCfg.h	(revision 877)
@@ -468,4 +468,7 @@
   Int  m_nCGSAdaptiveChroma;
 #endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Int  m_nCGSLutSizeRDO;
+#endif
 #endif 
 #endif //SVC_EXTENSION
Index: /branches/SHM-dev/source/App/TAppEncoder/TAppEncTop.cpp
===================================================================
--- /branches/SHM-dev/source/App/TAppEncoder/TAppEncTop.cpp	(revision 876)
+++ /branches/SHM-dev/source/App/TAppEncoder/TAppEncTop.cpp	(revision 877)
@@ -728,4 +728,7 @@
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
     m_acTEncTop[layer].setCGSAdaptChroma( m_nCGSAdaptiveChroma );
+#endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+    m_acTEncTop[layer].setCGSLutSizeRDO( m_nCGSLutSizeRDO );
 #endif
 #endif
Index: /branches/SHM-dev/source/Lib/TLibCommon/TCom3DAsymLUT.cpp
===================================================================
--- /branches/SHM-dev/source/Lib/TLibCommon/TCom3DAsymLUT.cpp	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibCommon/TCom3DAsymLUT.cpp	(revision 877)
@@ -76,4 +76,5 @@
 }
 
+
 Void TCom3DAsymLUT::xUpdatePartitioning( Int nCurOctantDepth , Int nCurYPartNumLog2 
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
@@ -101,4 +102,10 @@
 #endif
   m_nMappingOffset = 1 << ( m_nMappingShift - 1 );
+
+#if R0179_ENC_OPT_3DLUT_SIZE
+  m_nYSize = 1 << ( m_nCurOctantDepth + m_nCurYPartNumLog2 );
+  m_nUSize = 1 << m_nCurOctantDepth;
+  m_nVSize = 1 << m_nCurOctantDepth;
+#endif
 }
 
@@ -333,5 +340,9 @@
 Void TCom3DAsymLUT::xSaveCuboids( SCuboid *** pSrcCuboid )
 {
+#if R0179_ENC_OPT_3DLUT_SIZE
+  memcpy( m_pCuboid[0][0] , pSrcCuboid[0][0] , sizeof( SCuboid ) * getMaxYSize() * getMaxCSize() * getMaxCSize() );
+#else
   memcpy( m_pCuboid[0][0] , pSrcCuboid[0][0] , sizeof( SCuboid ) * m_nYSize * m_nUSize * m_nVSize );
+#endif 
 }
 
Index: /branches/SHM-dev/source/Lib/TLibCommon/TCom3DAsymLUT.h
===================================================================
--- /branches/SHM-dev/source/Lib/TLibCommon/TCom3DAsymLUT.h	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibCommon/TCom3DAsymLUT.h	(revision 877)
@@ -67,4 +67,8 @@
   Int   getAdaptChromaThresholdV()        { return m_nAdaptCThresholdV; }
 #endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Int   getMaxYSize() { return 1<<(m_nMaxOctantDepth+m_nMaxYPartNumLog2); }
+  Int   getMaxCSize() { return 1<<m_nMaxOctantDepth; }
+#endif 
 
   Void  colorMapping( TComPicYuv * pcPicSrc,  TComPicYuv * pcPicDst );
Index: /branches/SHM-dev/source/Lib/TLibCommon/TypeDef.h
===================================================================
--- /branches/SHM-dev/source/Lib/TLibCommon/TypeDef.h	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibCommon/TypeDef.h	(revision 877)
@@ -119,4 +119,5 @@
 #define R0179_CGS_SIZE_8x1x1             1      ///< JCTVC-R0179: allow CGS LUT size to be 8x1x1 as well 
 #define R0300_CGS_RES_COEFF_CODING       1      ///< JCTVC-R0300: improved residual coefficient coding for R0151
+#define R0179_ENC_OPT_3DLUT_SIZE         0      ///< JCTVC-R0179: RD decision based LUT size selection 
 #endif
 #define O0194_WEIGHTED_PREDICTION_CGS    1      ///< JCTVC-O0194: Weighted prediction for colour gamut scalability
Index: /branches/SHM-dev/source/Lib/TLibEncoder/TEnc3DAsymLUT.cpp
===================================================================
--- /branches/SHM-dev/source/Lib/TLibEncoder/TEnc3DAsymLUT.cpp	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibEncoder/TEnc3DAsymLUT.cpp	(revision 877)
@@ -14,4 +14,5 @@
   m_pColorInfoC = NULL;
   m_pEncCuboid = NULL;
+
   m_pBestEncCuboid = NULL;
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
@@ -29,4 +30,33 @@
   m_nPPSBit = 0;
   m_pDsOrigPic = NULL;
+#if R0179_ENC_OPT_3DLUT_SIZE
+  m_pMaxColorInfo = NULL;
+  m_pMaxColorInfoC = NULL;
+
+  
+  // fixed m_dDistFactor
+  Double dTmpFactor[3];   
+  dTmpFactor[I_SLICE] = 1.0;
+  dTmpFactor[P_SLICE] = 4./3.;
+  dTmpFactor[B_SLICE] = 1.5; 
+  for( Int iSliceType = 0; iSliceType < 3; iSliceType++) 
+  {
+    for(Int iLayer = 0; iLayer < MAX_TLAYER; iLayer++)
+    {
+      m_dDistFactor[iSliceType][iLayer] = dTmpFactor[iSliceType]*(Double)(1<<iLayer);      
+    }
+  }
+  // initialization with approximate number of bits to code the LUT
+  m_nNumLUTBits[0][0] = 200; // 1x1x1
+  m_nNumLUTBits[1][0] = 400; // 2x1x1  
+  m_nNumLUTBits[1][1] = 1500; // 2x2x2  
+  m_nNumLUTBits[2][0] = 800; // 4x1x1
+  m_nNumLUTBits[2][1] = 3200; // 4x2x2  
+  m_nNumLUTBits[2][2] = 8500; // 4x4x4  
+  m_nNumLUTBits[3][0] = 1200; // 8x1x1
+  m_nNumLUTBits[3][1] = 4500; // 8x2x2  
+  m_nNumLUTBits[3][2] = 10000; // 8x4x4  
+  m_nNumLUTBits[3][3] = 12000; // 8x8x8
+#endif
 }
 
@@ -47,4 +77,12 @@
   xAllocate3DArray( m_pEncCuboid , xGetYSize() , xGetUSize() , xGetVSize() );
   xAllocate3DArray( m_pBestEncCuboid , xGetYSize() , xGetUSize() , xGetVSize() );
+#if R0179_ENC_OPT_3DLUT_SIZE
+  xAllocate3DArray( m_pMaxColorInfo , xGetYSize() , xGetUSize() , xGetVSize() );
+  xAllocate3DArray( m_pMaxColorInfoC , xGetYSize() , xGetUSize() , xGetVSize() );
+
+  m_pEncCavlc = new TEncCavlc;
+  m_pBitstreamRedirect = new TComOutputBitstream;
+  m_pEncCavlc->setBitstream(m_pBitstreamRedirect);
+#endif
 }
 
@@ -55,4 +93,10 @@
   xFree3DArray( m_pEncCuboid );
   xFree3DArray( m_pBestEncCuboid );
+#if R0179_ENC_OPT_3DLUT_SIZE
+  xFree3DArray( m_pMaxColorInfo );
+  xFree3DArray( m_pMaxColorInfoC );
+  delete m_pBitstreamRedirect;
+  delete m_pEncCavlc;
+#endif 
   TCom3DAsymLUT::destroy();
 }
@@ -287,4 +331,173 @@
 
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
+#if R0179_ENC_OPT_3DLUT_SIZE
+Double TEnc3DAsymLUT::derive3DAsymLUT( TComSlice * pSlice , TComPic * pCurPic , UInt refLayerIdc , TEncCfg * pCfg , Bool bSignalPPS , Bool bElRapSliceTypeB, Double dFrameLambda )
+{
+  m_nLUTBitDepth = pCfg->getCGSLUTBit();
+
+  Int nBestAdaptCThresholdU = 1 << ( getInputBitDepthC() - 1 );
+  Int nBestAdaptCThresholdV = 1 << ( getInputBitDepthC() - 1 );
+  Int nAdaptCThresholdU, nAdaptCThresholdV;
+
+  Int nTmpLutBits[MAX_Y_SIZE][MAX_C_SIZE] ;
+  memset(nTmpLutBits, 0, sizeof(nTmpLutBits)); 
+
+  SLUTSize sMaxLutSize;  
+
+  // collect stats for the most partitions 
+  Int nCurYPartNumLog2 = 0 , nCurOctantDepth = 0; 
+  Int nMaxPartNumLog2 = xGetMaxPartNumLog2();
+
+  xxMapPartNum2DepthYPart( nMaxPartNumLog2 , nCurOctantDepth , nCurYPartNumLog2 ); 
+  xUpdatePartitioning( nCurOctantDepth , nCurYPartNumLog2, nBestAdaptCThresholdU, nBestAdaptCThresholdV ); 
+  xxCollectData( pCurPic , refLayerIdc );
+  xxCopyColorInfo(m_pMaxColorInfo, m_pColorInfo, m_pMaxColorInfoC, m_pColorInfoC); 
+ 
+  sMaxLutSize.iCPartNumLog2 = nCurOctantDepth; 
+  sMaxLutSize.iYPartNumLog2 = nCurOctantDepth + nCurYPartNumLog2; 
+
+  m_pBitstreamRedirect->clear();
+
+  // find the best partition based on RD cost 
+  Int i; 
+  Double dMinCost, dCurCost;
+
+  Int iBestLUTSizeIdx = 0;   
+  Int nBestResQuanBit = 0;
+  Double dCurError, dMinError; 
+  Int iNumBitsCurSize; 
+  Int iNumBitsCurSizeSave = m_pEncCavlc->getNumberOfWrittenBits(); 
+  Double dDistFactor = getDistFactor(pSlice->getSliceType(), pSlice->getDepth());
+
+  // check all LUT sizes 
+  xxGetAllLutSizes(pSlice);  
+  if (m_nTotalLutSizes == 0) // return if no valid size is found, LUT will not be updated
+  {
+    nCurOctantDepth = sMaxLutSize.iCPartNumLog2;
+    nCurYPartNumLog2 = sMaxLutSize.iYPartNumLog2-nCurOctantDepth; 
+    xUpdatePartitioning( nCurOctantDepth , nCurYPartNumLog2, nBestAdaptCThresholdU, nBestAdaptCThresholdV ); 
+    return MAX_DOUBLE; 
+  }
+
+  dMinCost = MAX_DOUBLE; dMinError = MAX_DOUBLE;
+  for (i = 0; i < m_nTotalLutSizes; i++)
+  {
+    // add up the stats
+    nCurOctantDepth = m_sLutSizes[i].iCPartNumLog2;
+    nCurYPartNumLog2 = m_sLutSizes[i].iYPartNumLog2-nCurOctantDepth; 
+    xUpdatePartitioning( nCurOctantDepth , nCurYPartNumLog2, nBestAdaptCThresholdU, nBestAdaptCThresholdV ); 
+    xxConsolidateData( &m_sLutSizes[i], &sMaxLutSize );
+  
+    dCurError = xxDeriveVertexes(nBestResQuanBit, m_pEncCuboid);
+
+    setResQuantBit( nBestResQuanBit );
+    xSaveCuboids( m_pEncCuboid ); 
+    m_pEncCavlc->xCode3DAsymLUT( this ); 
+    iNumBitsCurSize = m_pEncCavlc->getNumberOfWrittenBits();
+    dCurCost = dCurError/dDistFactor + dFrameLambda*(Double)(iNumBitsCurSize-iNumBitsCurSizeSave);  
+    nTmpLutBits[m_sLutSizes[i].iYPartNumLog2][m_sLutSizes[i].iCPartNumLog2] = iNumBitsCurSize-iNumBitsCurSizeSave; // store LUT size 
+    iNumBitsCurSizeSave = iNumBitsCurSize;
+    if(dCurCost < dMinCost )
+    {
+      SCuboid *** tmp = m_pBestEncCuboid;
+      m_pBestEncCuboid = m_pEncCuboid;
+      m_pEncCuboid = tmp;
+      dMinCost = dCurCost; 
+      dMinError = dCurError;
+      iBestLUTSizeIdx = i; 
+    }
+  }
+
+  nCurOctantDepth = m_sLutSizes[iBestLUTSizeIdx].iCPartNumLog2;
+  nCurYPartNumLog2 = m_sLutSizes[iBestLUTSizeIdx].iYPartNumLog2-nCurOctantDepth; 
+
+  xUpdatePartitioning( nCurOctantDepth , nCurYPartNumLog2, nBestAdaptCThresholdU, nBestAdaptCThresholdV ); 
+
+  Bool bUseNewColorInfo = false; 
+  if( pCfg->getCGSAdaptChroma() && nCurOctantDepth <= 1 ) // if the best size found so far has depth = 0 or 1, then check AdaptC U/V thresholds
+  {
+    nAdaptCThresholdU = ( Int )( m_dSumU / m_nNChroma + 0.5 );
+    nAdaptCThresholdV = ( Int )( m_dSumV / m_nNChroma + 0.5 );
+    if( !(nAdaptCThresholdU == nBestAdaptCThresholdU && nAdaptCThresholdV == nBestAdaptCThresholdV ) ) 
+    {
+      nCurOctantDepth = 1;
+      if( nCurOctantDepth + nCurYPartNumLog2 > getMaxYPartNumLog2()+getMaxOctantDepth() )
+        nCurYPartNumLog2 = getMaxYPartNumLog2()+getMaxOctantDepth()-nCurOctantDepth; 
+      xUpdatePartitioning( nCurOctantDepth , nCurYPartNumLog2 , nAdaptCThresholdU , nAdaptCThresholdV );
+      xxCollectData( pCurPic , refLayerIdc );
+
+      dCurError = xxDeriveVertexes( nBestResQuanBit , m_pEncCuboid ) ;
+      setResQuantBit( nBestResQuanBit );
+      xSaveCuboids( m_pEncCuboid ); 
+      m_pEncCavlc->xCode3DAsymLUT( this ); 
+      iNumBitsCurSize = m_pEncCavlc->getNumberOfWrittenBits();
+      dCurCost = dCurError/dDistFactor + dFrameLambda*(Double)(iNumBitsCurSize-iNumBitsCurSizeSave);  
+      iNumBitsCurSizeSave = iNumBitsCurSize;
+      if(dCurCost < dMinCost )
+      {
+        SCuboid *** tmp = m_pBestEncCuboid;
+        m_pBestEncCuboid = m_pEncCuboid;
+        m_pEncCuboid = tmp;
+        dMinCost = dCurCost; 
+        dMinError = dCurError;
+        nBestAdaptCThresholdU = nAdaptCThresholdU;
+        nBestAdaptCThresholdV = nAdaptCThresholdV;
+        bUseNewColorInfo = true; 
+      }
+    }
+  }
+
+  xUpdatePartitioning( nCurOctantDepth , nCurYPartNumLog2, nBestAdaptCThresholdU, nBestAdaptCThresholdV ); 
+
+  // check res_quant_bits only for the best table size and best U/V threshold
+  if( !bUseNewColorInfo ) 
+    xxConsolidateData( &m_sLutSizes[iBestLUTSizeIdx], &sMaxLutSize );
+
+  //    xxCollectData( pCurPic , refLayerIdc );
+  for( Int nResQuanBit = 1 ; nResQuanBit < 4 ; nResQuanBit++ )
+  {
+    dCurError = xxDeriveVertexes( nResQuanBit , m_pEncCuboid );
+
+    setResQuantBit( nResQuanBit );
+    xSaveCuboids( m_pEncCuboid ); 
+    m_pEncCavlc->xCode3DAsymLUT( this ); 
+    iNumBitsCurSize = m_pEncCavlc->getNumberOfWrittenBits();
+    dCurCost = dCurError/dDistFactor + dFrameLambda*(Double)(iNumBitsCurSize-iNumBitsCurSizeSave);   
+
+    iNumBitsCurSizeSave = iNumBitsCurSize;
+    if(dCurCost < dMinCost)
+    {
+      nBestResQuanBit = nResQuanBit;
+      SCuboid *** tmp = m_pBestEncCuboid;
+      m_pBestEncCuboid = m_pEncCuboid;
+      m_pEncCuboid = tmp;
+      dMinCost = dCurCost; 
+      dMinError = dCurError;
+    }
+    else
+    {
+      break;
+    }
+  }
+    
+  setResQuantBit( nBestResQuanBit );
+  xSaveCuboids( m_pBestEncCuboid );
+
+  // update LUT size stats 
+  for(Int iLutSizeY = 0; iLutSizeY < MAX_Y_SIZE; iLutSizeY++)
+  {
+    for(Int iLutSizeC = 0; iLutSizeC < MAX_C_SIZE; iLutSizeC++) 
+    {
+      if(nTmpLutBits[iLutSizeY][iLutSizeC] != 0) 
+        m_nNumLUTBits[iLutSizeY][iLutSizeC] =  (m_nNumLUTBits[iLutSizeY][iLutSizeC] + nTmpLutBits[iLutSizeY][iLutSizeC]*3+2)>>2; // update with new stats
+    }
+  }
+
+  // return cost rather than error
+  return( dMinCost );
+}
+#endif 
+
+
 Double TEnc3DAsymLUT::derive3DAsymLUT( TComSlice * pSlice , TComPic * pCurPic , UInt refLayerIdc , TEncCfg * pCfg , Bool bSignalPPS , Bool bElRapSliceTypeB )
 {
@@ -317,4 +530,5 @@
         break;
     }
+
     xUpdatePartitioning( nCurOctantDepth , nCurYPartNumLog2 , nAdaptCThresholdU , nAdaptCThresholdV );
     xxCollectData( pCurPic , refLayerIdc );
@@ -340,6 +554,8 @@
     }
   }
+
   setResQuantBit( nBestResQuanBit );
   xUpdatePartitioning( nBestOctantDepth , nBestYPartNumLog2 , nBestAdaptCThresholdU , nBestAdaptCThresholdV );
+
   xSaveCuboids( m_pBestEncCuboid );
   return( dMinError );
@@ -460,6 +676,11 @@
   Int nStrideILRY = pRecPic->getStride();
   Int nStrideILRC = pRecPic->getCStride();
+#if R0179_ENC_OPT_3DLUT_SIZE
+  xReset3DArray( m_pColorInfo  , getMaxYSize() , getMaxCSize() , getMaxCSize() );
+  xReset3DArray( m_pColorInfoC , getMaxYSize() , getMaxCSize() , getMaxCSize() );
+#else
   xReset3DArray( m_pColorInfo , xGetYSize() , xGetUSize() , xGetVSize() );
   xReset3DArray( m_pColorInfoC , xGetYSize() , xGetUSize() , xGetVSize() );
+#endif
 
   //alignment padding
@@ -693,5 +914,136 @@
   m_nPrevFrameCGSPartNumLog2[nSliceType][nSliceTempLevel] = getCurOctantDepth() * 3 + getCurYPartNumLog2();
 #endif
-}
-
-#endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Int nCurELFrameBit = pcSlice->getPic()->getFrameBit();
+  const Int nSliceType = pcSlice->getSliceType();
+  const Int nSliceTempLevel = pcSlice->getDepth();
+  m_nPrevELFrameBit[nSliceType][nSliceTempLevel] = m_nPrevELFrameBit[nSliceType][nSliceTempLevel] == 0 ? nCurELFrameBit:((m_nPrevELFrameBit[nSliceType][nSliceTempLevel]+nCurELFrameBit)>>1);
+#endif 
+}
+
+#if R0179_ENC_OPT_3DLUT_SIZE
+
+Void TEnc3DAsymLUT::xxGetAllLutSizes(TComSlice *pSlice)
+{
+  Int iMaxYPartNumLog2, iMaxCPartNumLog2; 
+  Int iCurYPartNumLog2, iCurCPartNumLog2; 
+  Int iMaxAddYPartNumLog2; 
+  Int iNumELFrameBits = m_nPrevELFrameBit[pSlice->getSliceType()][pSlice->getDepth()];
+
+  xxMapPartNum2DepthYPart( xGetMaxPartNumLog2() , iMaxCPartNumLog2 , iMaxYPartNumLog2 );
+  iMaxAddYPartNumLog2 = iMaxYPartNumLog2; 
+  iMaxYPartNumLog2 += iMaxCPartNumLog2; 
+
+  //m_sLutSizes[0].iYPartNumLog2 = iMaxYPartNumLog2; 
+  //m_sLutSizes[0].iCPartNumLog2 = iMaxCPartNumLog2; 
+  m_nTotalLutSizes = 0; 
+
+
+  for(iCurYPartNumLog2 = iMaxYPartNumLog2; iCurYPartNumLog2 >= 0; iCurYPartNumLog2--) 
+  {
+    for(iCurCPartNumLog2 = iMaxCPartNumLog2; iCurCPartNumLog2 >= 0; iCurCPartNumLog2--) 
+    {
+       // try more sizes
+      if(iCurCPartNumLog2 <= iCurYPartNumLog2  && 
+         (m_nNumLUTBits[iCurYPartNumLog2][iCurCPartNumLog2] < (iNumELFrameBits>>1)) && 
+         m_nTotalLutSizes < MAX_NUM_LUT_SIZES)
+      {
+        m_sLutSizes[m_nTotalLutSizes].iYPartNumLog2 = iCurYPartNumLog2; 
+        m_sLutSizes[m_nTotalLutSizes].iCPartNumLog2 = iCurCPartNumLog2; 
+        m_nTotalLutSizes ++; 
+      }
+    }
+  }
+
+}
+
+Void TEnc3DAsymLUT::xxCopyColorInfo( SColorInfo *** dst, SColorInfo *** src ,  SColorInfo *** dstC, SColorInfo *** srcC )
+{
+  Int yIdx, uIdx, vIdx; 
+
+  // copy from pColorInfo to pMaxColorInfo
+  for(yIdx = 0; yIdx < xGetYSize(); yIdx++)
+  {
+    for(uIdx = 0; uIdx < xGetUSize(); uIdx++)
+    {
+      for(vIdx = 0; vIdx < xGetVSize(); vIdx++)
+      {
+        dst [yIdx][uIdx][vIdx] = src [yIdx][uIdx][vIdx];
+        dstC[yIdx][uIdx][vIdx] = srcC[yIdx][uIdx][vIdx];
+      }
+    }
+  }
+}
+
+Void TEnc3DAsymLUT::xxAddColorInfo( Int yIdx, Int uIdx, Int vIdx, Int iYDiffLog2, Int iCDiffLog2 )
+{
+  SColorInfo & rCuboidColorInfo  = m_pColorInfo [yIdx][uIdx][vIdx];
+  SColorInfo & rCuboidColorInfoC = m_pColorInfoC[yIdx][uIdx][vIdx];
+  
+  for( Int i = 0; i < (1<<iYDiffLog2); i++)
+  {
+    for (Int j = 0; j < (1<<iCDiffLog2); j++)
+    {
+      for(Int k = 0; k < (1<<iCDiffLog2); k++)
+      {
+        rCuboidColorInfo  += m_pMaxColorInfo [(yIdx<<iYDiffLog2)+i][(uIdx<<iCDiffLog2)+j][(vIdx<<iCDiffLog2)+k];
+        rCuboidColorInfoC += m_pMaxColorInfoC[(yIdx<<iYDiffLog2)+i][(uIdx<<iCDiffLog2)+j][(vIdx<<iCDiffLog2)+k];
+      }
+    }
+  }
+}
+
+Void TEnc3DAsymLUT::xxConsolidateData( SLUTSize *pCurLUTSize, SLUTSize *pMaxLUTSize )
+{
+  Int yIdx, uIdx, vIdx; 
+  Int iYDiffLog2, iCDiffLog2;
+  Int nYSize = 1<< pMaxLUTSize->iYPartNumLog2;
+  Int nCSize = 1<< pMaxLUTSize->iCPartNumLog2;
+
+  iYDiffLog2 = pMaxLUTSize->iYPartNumLog2-pCurLUTSize->iYPartNumLog2;
+  iCDiffLog2 = pMaxLUTSize->iCPartNumLog2-pCurLUTSize->iCPartNumLog2;
+
+  //assert(pMaxLUTSize->iCPartNumLog2 >= pCurLUTSize->iCPartNumLog2 && pMaxLUTSize->iYPartNumLog2 >= pCurLUTSize->iYPartNumLog2); 
+  if (iYDiffLog2 == 0 && iCDiffLog2 == 0) // shouldn't have to do anything 
+  {
+    xxCopyColorInfo(m_pColorInfo, m_pMaxColorInfo, m_pColorInfoC, m_pMaxColorInfoC);
+    return; 
+  }
+
+  xReset3DArray( m_pColorInfo  ,  1<<pMaxLUTSize->iYPartNumLog2, 1<<pMaxLUTSize->iCPartNumLog2, 1<<pMaxLUTSize->iCPartNumLog2 );
+  xReset3DArray( m_pColorInfoC ,  1<<pMaxLUTSize->iYPartNumLog2, 1<<pMaxLUTSize->iCPartNumLog2, 1<<pMaxLUTSize->iCPartNumLog2 );
+
+  for(yIdx = 0; yIdx < nYSize; yIdx++)
+  {
+    for(uIdx = 0; uIdx < nCSize; uIdx++)
+    {
+      for(vIdx = 0; vIdx < nCSize; vIdx++)
+      {
+        const SColorInfo & rCuboidSrc   = m_pMaxColorInfo [yIdx][uIdx][vIdx];
+        const SColorInfo & rCuboidSrcC  = m_pMaxColorInfoC[yIdx][uIdx][vIdx];
+        
+        Int yIdx2, uIdx2, vIdx2; 
+        yIdx2 = yIdx>>iYDiffLog2; 
+        uIdx2 = uIdx>>iCDiffLog2;
+        vIdx2 = vIdx>>iCDiffLog2; 
+
+        m_pColorInfo [yIdx2][uIdx2][vIdx2] += rCuboidSrc;
+        m_pColorInfoC[yIdx2][uIdx2][vIdx2] += rCuboidSrcC;
+      }
+    }
+  }
+}
+
+Void TEnc3DAsymLUT::update3DAsymLUTParam( TEnc3DAsymLUT * pSrc )
+{
+  assert( pSrc->getMaxOctantDepth() == getMaxOctantDepth() && pSrc->getMaxYPartNumLog2() == getMaxYPartNumLog2() );
+  xUpdatePartitioning( pSrc->getCurOctantDepth() , pSrc->getCurYPartNumLog2() 
+#if R0151_CGS_3D_ASYMLUT_IMPROVE
+    , pSrc->getAdaptChromaThresholdU() , pSrc->getAdaptChromaThresholdV()
+#endif
+    );
+  setResQuantBit( pSrc->getResQuantBit() );
+}
+
+#endif 
+#endif
Index: /branches/SHM-dev/source/Lib/TLibEncoder/TEnc3DAsymLUT.h
===================================================================
--- /branches/SHM-dev/source/Lib/TLibEncoder/TEnc3DAsymLUT.h	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibEncoder/TEnc3DAsymLUT.h	(revision 877)
@@ -8,4 +8,10 @@
 #include "../TLibCommon/TComPic.h"
 #include "TEncCfg.h"
+#if R0179_ENC_OPT_3DLUT_SIZE
+#include "TEncCavlc.h"
+#define MAX_NUM_LUT_SIZES               10   // 4+3+2+1
+#define MAX_Y_SIZE                       4
+#define MAX_C_SIZE                       4
+#endif
 
 #if Q0048_CGS_3D_ASYMLUT
@@ -55,4 +61,11 @@
 }SColorInfo;
 
+#if R0179_ENC_OPT_3DLUT_SIZE
+typedef struct _LUTSize 
+{
+  Int iYPartNumLog2; 
+  Int iCPartNumLog2; 
+} SLUTSize; 
+#endif 
 
 class TEnc3DAsymLUT : public TCom3DAsymLUT
@@ -66,4 +79,9 @@
   Double derive3DAsymLUT( TComSlice * pSlice , TComPic * pCurPic , UInt refLayerIdc , TEncCfg * pCfg , Bool bSignalPPS , Bool bElRapSliceTypeB );
   Double estimateDistWithCur3DAsymLUT( TComPic * pCurPic , UInt refLayerIdc );
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Double getDistFactor( Int iSliceType, Int iLayer) { return m_dDistFactor[iSliceType][iLayer];}
+  Double derive3DAsymLUT( TComSlice * pSlice , TComPic * pCurPic , UInt refLayerIdc , TEncCfg * pCfg , Bool bSignalPPS , Bool bElRapSliceTypeB, Double dFrameLambda );
+  Void   update3DAsymLUTParam( TEnc3DAsymLUT * pSrc );
+#endif
 
   Void  updatePicCGSBits( TComSlice * pcSlice , Int nPPSBit );
@@ -75,4 +93,8 @@
   SColorInfo *** m_pColorInfo;
   SColorInfo *** m_pColorInfoC;
+#if R0179_ENC_OPT_3DLUT_SIZE
+  SColorInfo *** m_pMaxColorInfo;
+  SColorInfo *** m_pMaxColorInfoC;
+#endif 
   TComPicYuv* m_pDsOrigPic;
   SCuboid *** m_pEncCuboid;
@@ -92,4 +114,14 @@
   Int   m_nPPSBit;
   Int   m_nLUTBitDepth;
+#if R0179_ENC_OPT_3DLUT_SIZE
+
+  Double m_dDistFactor[3][MAX_TLAYER];         
+  Int    m_nNumLUTBits[MAX_Y_SIZE][MAX_C_SIZE]; 
+  Int    m_nPrevELFrameBit[3][MAX_TLAYER];   
+
+
+  Int   m_nTotalLutSizes;
+  SLUTSize m_sLutSizes[MAX_NUM_LUT_SIZES];
+#endif 
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
   Double m_dSumU;
@@ -97,4 +129,8 @@
   Int    m_nNChroma;
 #endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  TComOutputBitstream  *m_pBitstreamRedirect;
+  TEncCavlc *m_pEncCavlc;
+#endif 
 
 private:
@@ -111,4 +147,5 @@
   Int     xxCoeff2Vertex( Double a , Double b , Double c , Double d , Int y , Int u , Int v ) { return ( ( Int )( a * y + b * u + c * v + d + 0.5 ) ); }
   Void    xxCollectData( TComPic * pCurPic , UInt refLayerIdc );
+
   Double  xxDeriveVertexes( Int nResQuantBit , SCuboid *** pCurCuboid );
   inline Double  xxCalEstDist( Double N , Double Ys , Double Yy , Double Yu , Double Yv , Double ys , Double us , Double vs , Double yy , Double yu , Double yv , Double uu , Double uv , Double vv , Double YY ,
@@ -120,4 +157,10 @@
     Pel nP0 , Pel nP1 , Pel nP3 , Pel nP7 );
 #endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Void    xxConsolidateData( SLUTSize *pCurLUTSize, SLUTSize *pMaxLUTSize );
+  Void    xxGetAllLutSizes(TComSlice *pSlice);
+  Void    xxCopyColorInfo( SColorInfo *** dst, SColorInfo *** src ,  SColorInfo *** dstC, SColorInfo *** srcC ); 
+  Void    xxAddColorInfo( Int yIdx, Int uIdx, Int vIdx, Int iYDiffLog2, Int iCDiffLog2 );
+#endif 
 };
 
Index: /branches/SHM-dev/source/Lib/TLibEncoder/TEncCavlc.h
===================================================================
--- /branches/SHM-dev/source/Lib/TLibEncoder/TEncCavlc.h	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibEncoder/TEncCavlc.h	(revision 877)
@@ -170,7 +170,14 @@
 #endif
 #if Q0048_CGS_3D_ASYMLUT
+#if R0179_ENC_OPT_3DLUT_SIZE
+public:
+  Void xCode3DAsymLUT( TCom3DAsymLUT * pc3DAsymLUT );
+protected:
+  Void xCode3DAsymLUTOctant( TCom3DAsymLUT * pc3DAsymLUT , Int nDepth , Int yIdx , Int uIdx , Int vIdx , Int nLength );
+#else
 protected:
   Void xCode3DAsymLUT( TCom3DAsymLUT * pc3DAsymLUT );
   Void xCode3DAsymLUTOctant( TCom3DAsymLUT * pc3DAsymLUT , Int nDepth , Int yIdx , Int uIdx , Int vIdx , Int nLength );
+#endif
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
 #if R0300_CGS_RES_COEFF_CODING
Index: /branches/SHM-dev/source/Lib/TLibEncoder/TEncCfg.h
===================================================================
--- /branches/SHM-dev/source/Lib/TLibEncoder/TEncCfg.h	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibEncoder/TEncCfg.h	(revision 877)
@@ -369,4 +369,7 @@
 #if R0151_CGS_3D_ASYMLUT_IMPROVE
   Int  m_nCGSAdaptiveChroma;
+#endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Int  m_nCGSLutSizeRDO;
 #endif
 #endif
@@ -1004,4 +1007,8 @@
   Int       getCGSAdaptChroma()           { return m_nCGSAdaptiveChroma; }
 #endif
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Void      setCGSLutSizeRDO(Int n)      { m_nCGSLutSizeRDO = n;  }
+  Int       getCGSLutSizeRDO()           { return m_nCGSLutSizeRDO; }
+#endif
 #endif
 #endif
Index: /branches/SHM-dev/source/Lib/TLibEncoder/TEncGOP.cpp
===================================================================
--- /branches/SHM-dev/source/Lib/TLibEncoder/TEncGOP.cpp	(revision 876)
+++ /branches/SHM-dev/source/Lib/TLibEncoder/TEncGOP.cpp	(revision 877)
@@ -4032,6 +4032,10 @@
   if( m_layerId && pcSlice->getPPS()->getCGSFlag() )
   {
+#if R0179_ENC_OPT_3DLUT_SIZE
+      m_Enc3DAsymLUTPicUpdate.update3DAsymLUTParam( &m_Enc3DAsymLUTPPS );
+#else
     if( m_Enc3DAsymLUTPPS.getPPSBit() > 0 )
       m_Enc3DAsymLUTPicUpdate.copy3DAsymLUT( &m_Enc3DAsymLUTPPS );
+#endif
     m_Enc3DAsymLUTPicUpdate.updatePicCGSBits( pcSlice , m_Enc3DAsymLUTPPS.getPPSBit() );
   }
@@ -4756,5 +4760,27 @@
   m_Enc3DAsymLUTPPS.setPPSBit( 0 );
   Double dErrorUpdatedPPS = 0 , dErrorPPS = 0;
-  dErrorUpdatedPPS = m_Enc3DAsymLUTPicUpdate.derive3DAsymLUT( pSlice , pCurPic , refLayerIdc , pCfg , bSignalPPS , m_pcEncTop->getElRapSliceTypeB() );
+
+#if R0179_ENC_OPT_3DLUT_SIZE
+  Int nTLthres = m_pcCfg->getCGSLutSizeRDO() ? 2:7;
+  Double dFrameLambda; 
+#if FULL_NBIT
+  Int    SHIFT_QP = 12 + 6 * (pSlice->getBitDepthY() - 8);
+#else
+  Int    SHIFT_QP = 12; 
+#endif 
+  Int QP = pSlice->getSliceQp();
+
+  // set frame lambda
+  dFrameLambda = 0.68 * pow (2, (QP  - SHIFT_QP) / 3.0) * (m_pcCfg->getGOPSize() > 1 && pSlice->isInterB()? 2 : 1);
+
+  if(m_pcCfg->getCGSLutSizeRDO() == 1 && (!bSignalPPS && (pSlice->getDepth() < nTLthres))) 
+    dErrorUpdatedPPS = m_Enc3DAsymLUTPicUpdate.derive3DAsymLUT( pSlice , pCurPic , refLayerIdc , pCfg , bSignalPPS , m_pcEncTop->getElRapSliceTypeB(), dFrameLambda );
+  else if (pSlice->getDepth() >= nTLthres)
+    dErrorUpdatedPPS = MAX_DOUBLE;
+  else // if (m_pcCfg->getCGSLutSizeRDO() = 0 || bSignalPPS)
+#endif   
+    dErrorUpdatedPPS = m_Enc3DAsymLUTPicUpdate.derive3DAsymLUT( pSlice , pCurPic , refLayerIdc , pCfg , bSignalPPS , m_pcEncTop->getElRapSliceTypeB() );
+
+
   if( bSignalPPS )
   {
@@ -4764,7 +4790,29 @@
   else if( nCGSFlag )
   {
-    dErrorPPS = m_Enc3DAsymLUTPPS.estimateDistWithCur3DAsymLUT( pCurPic , refLayerIdc );
-    Double dFactor = pCfg->getIntraPeriod() == 1 ? 0.99 : 0.9;
-    pSlice->setCGSOverWritePPS( dErrorUpdatedPPS < dFactor * dErrorPPS );
+#if R0179_ENC_OPT_3DLUT_SIZE
+    if(pSlice->getNalUnitType() == NAL_UNIT_CODED_SLICE_RASL_R || pSlice->getNalUnitType() == NAL_UNIT_CODED_SLICE_RASL_N) 
+    {
+      pSlice->setCGSOverWritePPS( 0 ); 
+    }
+    else if (pSlice->getDepth() >= nTLthres) 
+    {
+      pSlice->setCGSOverWritePPS( 0 ); 
+    }
+    else
+    {
+#endif    
+      dErrorPPS = m_Enc3DAsymLUTPPS.estimateDistWithCur3DAsymLUT( pCurPic , refLayerIdc );
+      Double dFactor = pCfg->getIntraPeriod() == 1 ? 0.99 : 0.9;
+
+#if R0179_ENC_OPT_3DLUT_SIZE 
+      if( m_pcCfg->getCGSLutSizeRDO() )
+      {
+        dErrorPPS = dErrorPPS/m_Enc3DAsymLUTPicUpdate.getDistFactor(pSlice->getSliceType(), pSlice->getDepth()); 
+      }
+#endif
+      pSlice->setCGSOverWritePPS( dErrorUpdatedPPS < dFactor * dErrorPPS );
+#if R0179_ENC_OPT_3DLUT_SIZE
+    }
+#endif
     if( pSlice->getCGSOverWritePPS() )
     {
