Changeset 608 in 3DVCSoftware for trunk/source/Lib/TLibCommon/TComTrQuant.cpp
 Timestamp:
 1 Sep 2013, 22:47:26 (11 years ago)
 File:

 1 edited
Legend:
 Unmodified
 Added
 Removed

trunk/source/Lib/TLibCommon/TComTrQuant.cpp
r296 r608 4 4 * granted under this license. 5 5 * 6 * Copyright (c) 2010201 2, ITU/ISO/IEC6 * Copyright (c) 20102013, ITU/ISO/IEC 7 7 * All rights reserved. 8 8 * … … 117 117 118 118 Int cnt=0; 119 for( int u=1; u<=LEVEL_RANGE; u++)119 for(Int u=1; u<=LEVEL_RANGE; u++) 120 120 { 121 121 cnt += m_sliceNsamples[u] ; 122 122 } 123 123 124 if( !m_ bUseRDOQ )124 if( !m_useRDOQ ) 125 125 { 126 126 sliceQpused = qpBase; … … 190 190 * return void 191 191 */ 192 Void TComTrQuant::setQPforQuant( Int qpy, Bool bLowpass, SliceType eSliceType,TextType eTxtType, Int qpBdOffset, Int chromaQPOffset)192 Void TComTrQuant::setQPforQuant( Int qpy, TextType eTxtType, Int qpBdOffset, Int chromaQPOffset) 193 193 { 194 194 Int qpScaled; … … 200 200 else 201 201 { 202 qpScaled = Clip3( qpBdOffset, 5 1, qpy + chromaQPOffset );202 qpScaled = Clip3( qpBdOffset, 57, qpy + chromaQPOffset ); 203 203 204 204 if(qpScaled < 0) … … 208 208 else 209 209 { 210 qpScaled = g_aucChromaScale[ Clip3(0, 51, qpScaled)] + qpBdOffset;211 } 212 } 213 m_cQP.setQpParam( qpScaled , bLowpass, eSliceType);210 qpScaled = g_aucChromaScale[ qpScaled ] + qpBdOffset; 211 } 212 } 213 m_cQP.setQpParam( qpScaled ); 214 214 } 215 215 … … 222 222 * \param uiMode is Intra Prediction mode used in ModeDependent DCT/DST only 223 223 */ 224 void xTr( Pel *block, Int *coeff, UInt uiStride, UInt uiTrSize, UInt uiMode)224 void xTr(Int bitDepth, Pel *block, Int *coeff, UInt uiStride, UInt uiTrSize, UInt uiMode) 225 225 { 226 226 Int i,j,k,iSum; 227 227 Int tmp[32*32]; 228 const short *iT;228 const Short *iT; 229 229 UInt uiLog2TrSize = g_aucConvertToBit[ uiTrSize ] + 2; 230 230 … … 250 250 } 251 251 252 #if FULL_NBIT 253 int shift_1st = uiLog2TrSize  1 + g_uiBitDepth  8; // log2(N)  1 + g_uiBitDepth  8 254 #else 255 int shift_1st = uiLog2TrSize  1 + g_uiBitIncrement; // log2(N)  1 + g_uiBitIncrement 256 #endif 257 258 int add_1st = 1<<(shift_1st1); 259 int shift_2nd = uiLog2TrSize + 6; 260 int add_2nd = 1<<(shift_2nd1); 252 Int shift_1st = uiLog2TrSize  1 + bitDepth8; // log2(N)  1 + g_bitDepth8 253 Int add_1st = 1<<(shift_1st1); 254 Int shift_2nd = uiLog2TrSize + 6; 255 Int add_2nd = 1<<(shift_2nd1); 261 256 262 257 /* Horizontal transform */ … … 317 312 void xITr(Int *coeff, Pel *block, UInt uiStride, UInt uiTrSize, UInt uiMode) 318 313 { 319 int i,j,k,iSum;314 Int i,j,k,iSum; 320 315 Int tmp[32*32]; 321 const short *iT;316 const Short *iT; 322 317 323 318 if (uiTrSize==4) … … 342 337 } 343 338 344 int shift_1st = SHIFT_INV_1ST; 345 int add_1st = 1<<(shift_1st1); 346 #if FULL_NBIT 347 int shift_2nd = SHIFT_INV_2ND  ((short)g_uiBitDepth  8); 348 #else 349 int shift_2nd = SHIFT_INV_2ND  g_uiBitIncrement; 350 #endif 351 int add_2nd = 1<<(shift_2nd1); 339 Int shift_1st = SHIFT_INV_1ST; 340 Int add_1st = 1<<(shift_1st1); 341 Int shift_2nd = SHIFT_INV_2ND  g_bitDepth8; 342 Int add_2nd = 1<<(shift_2nd1); 352 343 if (uiTrSize==4) 353 344 { … … 407 398 */ 408 399 409 void partialButterfly4( short *src,short *dst,int shift, int line)410 { 411 int j;412 int E[2],O[2];413 int add = 1<<(shift1);400 void partialButterfly4(Short *src,Short *dst,Int shift, Int line) 401 { 402 Int j; 403 Int E[2],O[2]; 404 Int add = 1<<(shift1); 414 405 415 406 for (j=0; j<line; j++) … … 433 424 // Fast DST Algorithm. Full matrix multiplication for DST and Fast DST algorithm 434 425 // give identical results 435 void fastForwardDst( short *block,short *coeff,int shift) // input block, output coeff436 { 437 int i, c[4];438 int rnd_factor = 1<<(shift1);426 void fastForwardDst(Short *block,Short *coeff,Int shift) // input block, output coeff 427 { 428 Int i, c[4]; 429 Int rnd_factor = 1<<(shift1); 439 430 for (i=0; i<4; i++) 440 431 { … … 452 443 } 453 444 454 void fastInverseDst( short *tmp,short *block,int shift) // input tmp, output block455 { 456 int i, c[4];457 int rnd_factor = 1<<(shift1);445 void fastInverseDst(Short *tmp,Short *block,Int shift) // input tmp, output block 446 { 447 Int i, c[4]; 448 Int rnd_factor = 1<<(shift1); 458 449 for (i=0; i<4; i++) 459 450 { … … 471 462 } 472 463 473 void partialButterflyInverse4( short *src,short *dst,int shift, int line)474 { 475 int j;476 int E[2],O[2];477 int add = 1<<(shift1);464 void partialButterflyInverse4(Short *src,Short *dst,Int shift, Int line) 465 { 466 Int j; 467 Int E[2],O[2]; 468 Int add = 1<<(shift1); 478 469 479 470 for (j=0; j<line; j++) … … 497 488 498 489 499 void partialButterfly8( short *src,short *dst,int shift, int line)500 { 501 int j,k;502 int E[4],O[4];503 int EE[2],EO[2];504 int add = 1<<(shift1);490 void partialButterfly8(Short *src,Short *dst,Int shift, Int line) 491 { 492 Int j,k; 493 Int E[4],O[4]; 494 Int EE[2],EO[2]; 495 Int add = 1<<(shift1); 505 496 506 497 for (j=0; j<line; j++) … … 534 525 535 526 536 void partialButterflyInverse8( short *src,short *dst,int shift, int line)537 { 538 int j,k;539 int E[4],O[4];540 int EE[2],EO[2];541 int add = 1<<(shift1);527 void partialButterflyInverse8(Short *src,Short *dst,Int shift, Int line) 528 { 529 Int j,k; 530 Int E[4],O[4]; 531 Int EE[2],EO[2]; 532 Int add = 1<<(shift1); 542 533 543 534 for (j=0; j<line; j++) … … 570 561 571 562 572 void partialButterfly16( short *src,short *dst,int shift, int line)573 { 574 int j,k;575 int E[8],O[8];576 int EE[4],EO[4];577 int EEE[2],EEO[2];578 int add = 1<<(shift1);563 void partialButterfly16(Short *src,Short *dst,Int shift, Int line) 564 { 565 Int j,k; 566 Int E[8],O[8]; 567 Int EE[4],EO[4]; 568 Int EEE[2],EEO[2]; 569 Int add = 1<<(shift1); 579 570 580 571 for (j=0; j<line; j++) … … 621 612 622 613 623 void partialButterflyInverse16( short *src,short *dst,int shift, int line)624 { 625 int j,k;626 int E[8],O[8];627 int EE[4],EO[4];628 int EEE[2],EEO[2];629 int add = 1<<(shift1);614 void partialButterflyInverse16(Short *src,Short *dst,Int shift, Int line) 615 { 616 Int j,k; 617 Int E[8],O[8]; 618 Int EE[4],EO[4]; 619 Int EEE[2],EEO[2]; 620 Int add = 1<<(shift1); 630 621 631 622 for (j=0; j<line; j++) … … 668 659 669 660 670 void partialButterfly32( short *src,short *dst,int shift, int line)671 { 672 int j,k;673 int E[16],O[16];674 int EE[8],EO[8];675 int EEE[4],EEO[4];676 int EEEE[2],EEEO[2];677 int add = 1<<(shift1);661 void partialButterfly32(Short *src,Short *dst,Int shift, Int line) 662 { 663 Int j,k; 664 Int E[16],O[16]; 665 Int EE[8],EO[8]; 666 Int EEE[4],EEO[4]; 667 Int EEEE[2],EEEO[2]; 668 Int add = 1<<(shift1); 678 669 679 670 for (j=0; j<line; j++) … … 729 720 730 721 731 void partialButterflyInverse32( short *src,short *dst,int shift, int line)732 { 733 int j,k;734 int E[16],O[16];735 int EE[8],EO[8];736 int EEE[4],EEO[4];737 int EEEE[2],EEEO[2];738 int add = 1<<(shift1);722 void partialButterflyInverse32(Short *src,Short *dst,Int shift, Int line) 723 { 724 Int j,k; 725 Int E[16],O[16]; 726 Int EE[8],EO[8]; 727 Int EEE[4],EEO[4]; 728 Int EEEE[2],EEEO[2]; 729 Int add = 1<<(shift1); 739 730 740 731 for (j=0; j<line; j++) … … 787 778 } 788 779 789 790 780 /** MxN forward transform (2D) 791 781 * \param block input data (residual) … … 794 784 * \param iHeight input data (height of transform) 795 785 */ 796 void xTrMxN(short *block,short *coeff, int iWidth, int iHeight, UInt uiMode) 797 { 798 #if FULL_NBIT 799 int shift_1st = g_aucConvertToBit[iWidth] + 1 + g_uiBitDepth  8; // log2(iWidth)  1 + g_uiBitDepth  8 800 #else 801 int shift_1st = g_aucConvertToBit[iWidth] + 1 + g_uiBitIncrement; // log2(iWidth)  1 + g_uiBitIncrement 802 #endif 803 int shift_2nd = g_aucConvertToBit[iHeight] + 8; // log2(iHeight) + 6 804 805 short tmp[ 64 * 64 ]; 806 807 if( iWidth == 16 && iHeight == 4) 808 { 809 partialButterfly16( block, tmp, shift_1st, iHeight ); 810 partialButterfly4( tmp, coeff, shift_2nd, iWidth ); 811 } 812 else if( iWidth == 32 && iHeight == 8 ) 813 { 814 partialButterfly32( block, tmp, shift_1st, iHeight ); 815 partialButterfly8( tmp, coeff, shift_2nd, iWidth ); 816 } 817 else if( iWidth == 4 && iHeight == 16) 818 { 819 partialButterfly4( block, tmp, shift_1st, iHeight ); 820 partialButterfly16( tmp, coeff, shift_2nd, iWidth ); 821 } 822 else if( iWidth == 8 && iHeight == 32 ) 823 { 824 partialButterfly8( block, tmp, shift_1st, iHeight ); 825 partialButterfly32( tmp, coeff, shift_2nd, iWidth ); 826 } 827 else if( iWidth == 4 && iHeight == 4) 828 { 829 if (uiMode != REG_DCT && (!uiMode  (uiMode>=2 && uiMode <= 25))) // Check for DCT or DST 786 void xTrMxN(Int bitDepth, Short *block,Short *coeff, Int iWidth, Int iHeight, UInt uiMode) 787 { 788 Int shift_1st = g_aucConvertToBit[iWidth] + 1 + bitDepth8; // log2(iWidth)  1 + g_bitDepth  8 789 Int shift_2nd = g_aucConvertToBit[iHeight] + 8; // log2(iHeight) + 6 790 791 Short tmp[ 64 * 64 ]; 792 793 if( iWidth == 4 && iHeight == 4) 794 { 795 if (uiMode != REG_DCT) 830 796 { 831 797 fastForwardDst(block,tmp,shift_1st); // Forward DST BY FAST ALGORITHM, block input, tmp output 832 } 833 else 798 fastForwardDst(tmp,coeff,shift_2nd); // Forward DST BY FAST ALGORITHM, tmp input, coeff output 799 } 800 else 834 801 { 835 802 partialButterfly4(block, tmp, shift_1st, iHeight); 836 }837 if (uiMode != REG_DCT && (!uiMode  (uiMode>=11 && uiMode <= 34))) // Check for DCT or DST838 {839 fastForwardDst(tmp,coeff,shift_2nd); // Forward DST BY FAST ALGORITHM, tmp input, coeff output840 }841 else842 {843 803 partialButterfly4(tmp, coeff, shift_2nd, iWidth); 844 } 804 } 805 845 806 } 846 807 else if( iWidth == 8 && iHeight == 8) … … 866 827 * \param iHeight input data (height of transform) 867 828 */ 868 void xITrMxN(short *coeff,short *block, int iWidth, int iHeight, UInt uiMode) 869 { 870 int shift_1st = SHIFT_INV_1ST; 871 #if FULL_NBIT 872 int shift_2nd = SHIFT_INV_2ND  ((short)g_uiBitDepth  8); 873 #else 874 int shift_2nd = SHIFT_INV_2ND  g_uiBitIncrement; 875 #endif 876 877 short tmp[ 64*64]; 878 if( iWidth == 16 && iHeight == 4) 879 { 880 partialButterflyInverse4(coeff,tmp,shift_1st,iWidth); 881 partialButterflyInverse16(tmp,block,shift_2nd,iHeight); 882 } 883 else if( iWidth == 32 && iHeight == 8) 884 { 885 partialButterflyInverse8(coeff,tmp,shift_1st,iWidth); 886 partialButterflyInverse32(tmp,block,shift_2nd,iHeight); 887 } 888 else if( iWidth == 4 && iHeight == 16) 889 { 890 partialButterflyInverse16(coeff,tmp,shift_1st,iWidth); 891 partialButterflyInverse4(tmp,block,shift_2nd,iHeight); 892 } 893 else if( iWidth == 8 && iHeight == 32) 894 { 895 partialButterflyInverse32(coeff,tmp,shift_1st,iWidth); 896 partialButterflyInverse8(tmp,block,shift_2nd,iHeight); 897 } 898 else if( iWidth == 4 && iHeight == 4) 899 { 900 if (uiMode != REG_DCT && (!uiMode  (uiMode>=11 && uiMode <= 34))) // Check for DCT or DST 829 void xITrMxN(Int bitDepth, Short *coeff,Short *block, Int iWidth, Int iHeight, UInt uiMode) 830 { 831 Int shift_1st = SHIFT_INV_1ST; 832 Int shift_2nd = SHIFT_INV_2ND  (bitDepth8); 833 834 Short tmp[ 64*64]; 835 if( iWidth == 4 && iHeight == 4) 836 { 837 if (uiMode != REG_DCT) 901 838 { 902 839 fastInverseDst(coeff,tmp,shift_1st); // Inverse DST by FAST Algorithm, coeff input, tmp output 840 fastInverseDst(tmp,block,shift_2nd); // Inverse DST by FAST Algorithm, tmp input, coeff output 903 841 } 904 842 else 905 843 { 906 partialButterflyInverse4(coeff,tmp,shift_1st,iWidth); 907 } 908 if (uiMode != REG_DCT && (!uiMode  (uiMode>=2 && uiMode <= 25))) // Check for DCT or DST 909 { 910 fastInverseDst(tmp,block,shift_2nd); // Inverse DST by FAST Algorithm, tmp input, coeff output 911 } 912 else 913 { 844 partialButterflyInverse4(coeff,tmp,shift_1st,iWidth); 914 845 partialButterflyInverse4(tmp,block,shift_2nd,iHeight); 915 } 846 } 916 847 } 917 848 else if( iWidth == 8 && iHeight == 8) … … 935 866 936 867 // To minimize the distortion only. No rate is considered. 937 Void TComTrQuant::signBitHidingHDQ( TComDataCU* pcCU, TCoeff* pQCoef, TCoeff* pCoef, UInt const *scan, Int* deltaU, Int width, Int height ) 938 { 939 Int tsig = pcCU>getSlice()>getPPS()>getTSIG() ; 868 Void TComTrQuant::signBitHidingHDQ( TCoeff* pQCoef, TCoeff* pCoef, UInt const *scan, Int* deltaU, Int width, Int height ) 869 { 940 870 Int lastCG = 1; 941 871 Int absSum = 0 ; … … 975 905 lastCG = 1 ; 976 906 } 977 978 if( lastNZPosInCGfirstNZPosInCG>= tsig)907 908 if( lastNZPosInCGfirstNZPosInCG>=SBH_THRESHOLD ) 979 909 { 980 910 UInt signbit = (pQCoef[scan[subPos+firstNZPosInCG]]>0?0:1) ; … … 1079 1009 #endif 1080 1010 Int iAdd = 0; 1081 1082 if ( m_bUseRDOQ && (eTType == TEXT_LUMA  RDOQ_CHROMA) ) 1011 1012 Bool useRDOQ = pcCU>getTransformSkip(uiAbsPartIdx,eTType) ? m_useRDOQTS:m_useRDOQ; 1013 if ( useRDOQ && (eTType == TEXT_LUMA  RDOQ_CHROMA)) 1083 1014 { 1084 1015 #if ADAPTIVE_QP_SELECTION … … 1093 1024 1094 1025 UInt scanIdx = pcCU>getCoefScanIdx(uiAbsPartIdx, iWidth, eTType==TEXT_LUMA, pcCU>isIntra(uiAbsPartIdx)); 1095 if (scanIdx == SCAN_ZIGZAG) 1096 { 1097 scanIdx = SCAN_DIAG; 1098 } 1099 1100 if (iWidth != iHeight) 1101 { 1102 scanIdx = SCAN_DIAG; 1103 } 1104 1105 const UInt * scan; 1106 if (iWidth == iHeight) 1107 { 1108 scan = g_auiSigLastScan[ scanIdx ][ log2BlockSize  1 ]; 1109 } 1110 else 1111 { 1112 scan = g_sigScanNSQT[ log2BlockSize  2 ]; 1113 } 1114 1026 const UInt *scan = g_auiSigLastScan[ scanIdx ][ log2BlockSize  1 ]; 1027 1115 1028 Int deltaU[32*32] ; 1116 1029 … … 1128 1041 else 1129 1042 { 1130 qpScaled = Clip3( qpBDOffset, 51, iQpBase); 1043 Int chromaQPOffset; 1044 if(eTType == TEXT_CHROMA_U) 1045 { 1046 chromaQPOffset = pcCU>getSlice()>getPPS()>getChromaCbQpOffset() + pcCU>getSlice()>getSliceQpDeltaCb(); 1047 } 1048 else 1049 { 1050 chromaQPOffset = pcCU>getSlice()>getPPS()>getChromaCrQpOffset() + pcCU>getSlice()>getSliceQpDeltaCr(); 1051 } 1052 iQpBase = iQpBase + chromaQPOffset; 1053 1054 qpScaled = Clip3( qpBDOffset, 57, iQpBase); 1131 1055 1132 1056 if(qpScaled < 0) … … 1136 1060 else 1137 1061 { 1138 qpScaled = g_aucChromaScale[ Clip3(0, 51, qpScaled)] + qpBDOffset;1139 } 1140 } 1141 cQpBase.setQpParam(qpScaled , false, pcCU>getSlice()>getSliceType());1062 qpScaled = g_aucChromaScale[ qpScaled ] + qpBDOffset; 1063 } 1064 } 1065 cQpBase.setQpParam(qpScaled); 1142 1066 #endif 1143 1144 Bool bNonSqureFlag = ( iWidth != iHeight );1145 UInt dir = SCALING_LIST_SQT;1146 if( bNonSqureFlag )1147 {1148 dir = ( iWidth < iHeight )? SCALING_LIST_VER: SCALING_LIST_HOR;1149 UInt uiWidthBit = g_aucConvertToBit[ iWidth ] + 2;1150 UInt uiHeightBit = g_aucConvertToBit[ iHeight ] + 2;1151 iWidth = 1 << ( ( uiWidthBit + uiHeightBit) >> 1 );1152 iHeight = iWidth;1153 }1154 1067 1155 1068 UInt uiLog2TrSize = g_aucConvertToBit[ iWidth ] + 2; … … 1157 1070 assert(scalingListType < 6); 1158 1071 Int *piQuantCoeff = 0; 1159 piQuantCoeff = getQuantCoeff(scalingListType,m_cQP.m_iRem,uiLog2TrSize2, dir); 1160 1161 #if FULL_NBIT 1162 UInt uiBitDepth = g_uiBitDepth; 1163 #else 1164 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 1165 #endif 1166 UInt iTransformShift = MAX_TR_DYNAMIC_RANGE  uiBitDepth  uiLog2TrSize; // Represents scaling through forward transform 1167 Int iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift; // Right shift of nonRDOQ quantizer; level = (coeff*uiQ + offset)>>q_bits 1168 1169 iAdd = (pcCU>getSlice()>getSliceType()==I_SLICE ? 171 : 85) << (iQBits9); 1072 piQuantCoeff = getQuantCoeff(scalingListType,m_cQP.m_iRem,uiLog2TrSize2); 1073 1074 UInt uiBitDepth = eTType == TEXT_LUMA ? g_bitDepthY : g_bitDepthC; 1075 Int iTransformShift = MAX_TR_DYNAMIC_RANGE  uiBitDepth  uiLog2TrSize; // Represents scaling through forward transform 1170 1076 1171 1077 #if ADAPTIVE_QP_SELECTION 1172 iQBits = QUANT_SHIFT + cQpBase.m_iPer + iTransformShift;1078 Int iQBits = QUANT_SHIFT + cQpBase.m_iPer + iTransformShift; 1173 1079 iAdd = (pcCU>getSlice()>getSliceType()==I_SLICE ? 171 : 85) << (iQBits9); 1174 1080 Int iQBitsC = QUANT_SHIFT + cQpBase.m_iPer + iTransformShift  ARL_C_PRECISION; 1175 1081 Int iAddC = 1 << (iQBitsC1); 1082 #else 1083 Int iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift; // Right shift of nonRDOQ quantizer; level = (coeff*uiQ + offset)>>q_bits 1084 iAdd = (pcCU>getSlice()>getSliceType()==I_SLICE ? 171 : 85) << (iQBits9); 1176 1085 #endif 1177 1086 … … 1195 1104 #else 1196 1105 iLevel = ((Int64)abs(iLevel) * piQuantCoeff[uiBlockPos] + iAdd ) >> iQBits; 1197 deltaU[uiBlockPos] = (Int)( ((Int64)abs( iLevel) * piQuantCoeff[uiBlockPos]  (iLevel<<iQBits) )>> qBits8 );1106 deltaU[uiBlockPos] = (Int)( ((Int64)abs(piCoef[uiBlockPos]) * piQuantCoeff[uiBlockPos]  (iLevel<<iQBits) )>> qBits8 ); 1198 1107 #endif 1199 1108 uiAcSum += iLevel; … … 1205 1114 if(uiAcSum>=2) 1206 1115 { 1207 signBitHidingHDQ( p cCU, piQCoef, piCoef, scan, deltaU, iWidth, iHeight ) ;1116 signBitHidingHDQ( piQCoef, piCoef, scan, deltaU, iWidth, iHeight ) ; 1208 1117 } 1209 1118 } … … 1213 1122 } 1214 1123 1215 Void TComTrQuant::xDeQuant( const TCoeff* pSrc, Int* pDes, Int iWidth, Int iHeight, Int scalingListType )1124 Void TComTrQuant::xDeQuant(Int bitDepth, const TCoeff* pSrc, Int* pDes, Int iWidth, Int iHeight, Int scalingListType ) 1216 1125 { 1217 1126 1218 1127 const TCoeff* piQCoef = pSrc; 1219 1128 Int* piCoef = pDes; 1220 UInt dir = SCALING_LIST_SQT; 1221 if( iWidth != iHeight ) 1222 { 1223 dir = ( iWidth < iHeight )? SCALING_LIST_VER: SCALING_LIST_HOR; 1224 UInt uiWidthBit = g_aucConvertToBit[ iWidth ] + 2; 1225 UInt uiHeightBit = g_aucConvertToBit[ iHeight ] + 2; 1226 iWidth = 1 << ( ( uiWidthBit + uiHeightBit) >> 1 ); 1227 iHeight = iWidth; 1228 } 1229 1129 1230 1130 if ( iWidth > (Int)m_uiMaxTrSize ) 1231 1131 { … … 1237 1137 UInt uiLog2TrSize = g_aucConvertToBit[ iWidth ] + 2; 1238 1138 1239 #if FULL_NBIT 1240 UInt uiBitDepth = g_uiBitDepth; 1241 #else 1242 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 1243 #endif 1244 UInt iTransformShift = MAX_TR_DYNAMIC_RANGE  uiBitDepth  uiLog2TrSize; 1139 Int iTransformShift = MAX_TR_DYNAMIC_RANGE  bitDepth  uiLog2TrSize; 1140 1245 1141 iShift = QUANT_IQUANT_SHIFT  QUANT_SHIFT  iTransformShift; 1246 1142 1247 1143 TCoeff clipQCoef; 1248 const Int bitRange = min( 15, ( Int )( 12 + uiLog2TrSize + uiBitDepth  m_cQP.m_iPer) );1249 const Int levelLimit = 1 << bitRange;1250 1144 1251 1145 if(getUseScalingList()) 1252 1146 { 1253 1147 iShift += 4; 1148 Int *piDequantCoef = getDequantCoeff(scalingListType,m_cQP.m_iRem,uiLog2TrSize2); 1149 1254 1150 if(iShift > m_cQP.m_iPer) 1255 1151 { 1256 1152 iAdd = 1 << (iShift  m_cQP.m_iPer  1); 1257 } 1258 else 1259 { 1260 iAdd = 0; 1261 } 1262 Int *piDequantCoef = getDequantCoeff(scalingListType,m_cQP.m_iRem,uiLog2TrSize2,dir); 1263 1264 if(iShift > m_cQP.m_iPer) 1265 { 1153 1266 1154 for( Int n = 0; n < iWidth*iHeight; n++ ) 1267 1155 { … … 1275 1163 for( Int n = 0; n < iWidth*iHeight; n++ ) 1276 1164 { 1277 clipQCoef = Clip3(  levelLimit, levelLimit  1, piQCoef[n] );1278 iCoeffQ = (clipQCoef * piDequantCoef[n]) << (m_cQP.m_iPer  iShift);1279 piCoef[n] = Clip3( 32768,32767,iCoeffQ);1165 clipQCoef = Clip3( 32768, 32767, piQCoef[n] ); 1166 iCoeffQ = Clip3( 32768, 32767, clipQCoef * piDequantCoef[n] ); // Clip to avoid possible overflow in following shift left operation 1167 piCoef[n] = Clip3( 32768, 32767, iCoeffQ << ( m_cQP.m_iPer  iShift ) ); 1280 1168 } 1281 1169 } … … 1295 1183 } 1296 1184 1297 Void TComTrQuant::init( UInt uiMaxWidth, UInt uiMaxHeight, UInt uiMaxTrSize, Int iSymbolMode, UInt *aTableLP4, UInt *aTableLP8, UInt *aTableLastPosVlcIndex, 1298 Bool bUseRDOQ, Bool bEnc 1185 Void TComTrQuant::init( UInt uiMaxTrSize, 1186 Bool bUseRDOQ, 1187 Bool bUseRDOQTS, 1188 Bool bEnc, Bool useTransformSkipFast 1299 1189 #if ADAPTIVE_QP_SELECTION 1300 1190 , Bool bUseAdaptQpSelect … … 1304 1194 m_uiMaxTrSize = uiMaxTrSize; 1305 1195 m_bEnc = bEnc; 1306 m_bUseRDOQ = bUseRDOQ; 1196 m_useRDOQ = bUseRDOQ; 1197 m_useRDOQTS = bUseRDOQTS; 1307 1198 #if ADAPTIVE_QP_SELECTION 1308 1199 m_bUseAdaptQpSelect = bUseAdaptQpSelect; 1309 1200 #endif 1201 m_useTransformSkipFast = useTransformSkipFast; 1310 1202 } 1311 1203 1312 1204 Void TComTrQuant::transformNxN( TComDataCU* pcCU, 1313 1314 1315 1205 Pel* pcResidual, 1206 UInt uiStride, 1207 TCoeff* rpcCoeff, 1316 1208 #if ADAPTIVE_QP_SELECTION 1317 1209 Int*& rpcArlCoeff, 1318 1210 #endif 1319 UInt uiWidth, 1320 UInt uiHeight, 1321 UInt& uiAbsSum, 1322 TextType eTType, 1323 UInt uiAbsPartIdx ) 1324 { 1325 #if LOSSLESS_CODING 1326 if((m_cQP.qp() == 0) && (pcCU>getSlice()>getSPS()>getUseLossless())) 1211 UInt uiWidth, 1212 UInt uiHeight, 1213 UInt& uiAbsSum, 1214 TextType eTType, 1215 UInt uiAbsPartIdx, 1216 Bool useTransformSkip 1217 ) 1218 { 1219 if (pcCU>getCUTransquantBypass(uiAbsPartIdx)) 1327 1220 { 1328 1221 uiAbsSum=0; … … 1337 1230 return; 1338 1231 } 1339 #endif1340 1232 UInt uiMode; //luma intra pred 1341 1233 if(eTType == TEXT_LUMA && pcCU>getPredictionMode(uiAbsPartIdx) == MODE_INTRA ) … … 1350 1242 uiAbsSum = 0; 1351 1243 assert( (pcCU>getSlice()>getSPS()>getMaxTrSize() >= uiWidth) ); 1352 1353 xT( uiMode, pcResidual, uiStride, m_plTempCoeff, uiWidth, uiHeight ); 1244 Int bitDepth = eTType == TEXT_LUMA ? g_bitDepthY : g_bitDepthC; 1245 if(useTransformSkip) 1246 { 1247 xTransformSkip(bitDepth, pcResidual, uiStride, m_plTempCoeff, uiWidth, uiHeight ); 1248 } 1249 else 1250 { 1251 xT(bitDepth, uiMode, pcResidual, uiStride, m_plTempCoeff, uiWidth, uiHeight ); 1252 } 1354 1253 xQuant( pcCU, m_plTempCoeff, rpcCoeff, 1355 1254 #if ADAPTIVE_QP_SELECTION … … 1359 1258 } 1360 1259 1361 #if LOSSLESS_CODING 1362 Void TComTrQuant::invtransformNxN( TComDataCU* pcCU, TextType eText, UInt uiMode,Pel* rpcResidual, UInt uiStride, TCoeff* pcCoeff, UInt uiWidth, UInt uiHeight, Int scalingListType) 1363 #else 1364 Void TComTrQuant::invtransformNxN( TextType eText, UInt uiMode,Pel*& rpcResidual, UInt uiStride, TCoeff* pcCoeff, UInt uiWidth, UInt uiHeight, Int scalingListType) 1365 #endif 1366 { 1367 #if LOSSLESS_CODING 1368 if((m_cQP.qp() == 0) && (pcCU>getSlice()>getSPS()>getUseLossless())) 1260 Void TComTrQuant::invtransformNxN( Bool transQuantBypass, TextType eText, UInt uiMode,Pel* rpcResidual, UInt uiStride, TCoeff* pcCoeff, UInt uiWidth, UInt uiHeight, Int scalingListType, Bool useTransformSkip ) 1261 { 1262 if(transQuantBypass) 1369 1263 { 1370 1264 for (UInt k = 0; k<uiHeight; k++) … … 1377 1271 return; 1378 1272 } 1379 #endif 1380 xDeQuant( pcCoeff, m_plTempCoeff, uiWidth, uiHeight, scalingListType); 1381 xIT( uiMode, m_plTempCoeff, rpcResidual, uiStride, uiWidth, uiHeight ); 1273 Int bitDepth = eText == TEXT_LUMA ? g_bitDepthY : g_bitDepthC; 1274 xDeQuant(bitDepth, pcCoeff, m_plTempCoeff, uiWidth, uiHeight, scalingListType); 1275 if(useTransformSkip == true) 1276 { 1277 xITransformSkip(bitDepth, m_plTempCoeff, rpcResidual, uiStride, uiWidth, uiHeight ); 1278 } 1279 else 1280 { 1281 xIT(bitDepth, uiMode, m_plTempCoeff, rpcResidual, uiStride, uiWidth, uiHeight ); 1282 } 1382 1283 } 1383 1284 … … 1387 1288 { 1388 1289 return; 1389 } 1390 1391 UInt uiLumaTrMode, uiChromaTrMode; 1392 pcCU>convertTransIdx( uiAbsPartIdx, pcCU>getTransformIdx( uiAbsPartIdx ), uiLumaTrMode, uiChromaTrMode ); 1393 const UInt uiStopTrMode = eTxt == TEXT_LUMA ? uiLumaTrMode : uiChromaTrMode; 1394 1395 if( uiTrMode == uiStopTrMode ) 1290 } 1291 const UInt stopTrMode = pcCU>getTransformIdx( uiAbsPartIdx ); 1292 1293 if( uiTrMode == stopTrMode ) 1396 1294 { 1397 1295 UInt uiDepth = pcCU>getDepth( uiAbsPartIdx ) + uiTrMode; … … 1408 1306 } 1409 1307 Pel* pResi = rpcResidual + uiAddr; 1410 if( pcCU>useNonSquareTrans( uiTrMode, uiAbsPartIdx ) )1411 {1412 Int trWidth = uiWidth;1413 Int trHeight = uiHeight;1414 pcCU>getNSQTSize( uiTrMode, uiAbsPartIdx, trWidth, trHeight );1415 1416 uiWidth = trWidth;1417 uiHeight = trHeight;1418 }1419 1308 Int scalingListType = (pcCU>isIntra(uiAbsPartIdx) ? 0 : 3) + g_eTTable[(Int)eTxt]; 1420 1309 assert(scalingListType < 6); 1421 #if LOSSLESS_CODING 1422 invtransformNxN( pcCU, eTxt, REG_DCT, pResi, uiStride, rpcCoeff, uiWidth, uiHeight, scalingListType ); 1423 #else 1424 invtransformNxN( eTxt, REG_DCT, pResi, uiStride, rpcCoeff, uiWidth, uiHeight, scalingListType ); 1425 #endif 1310 invtransformNxN( pcCU>getCUTransquantBypass(uiAbsPartIdx), eTxt, REG_DCT, pResi, uiStride, rpcCoeff, uiWidth, uiHeight, scalingListType, pcCU>getTransformSkip(uiAbsPartIdx, eTxt) ); 1426 1311 } 1427 1312 else … … 1431 1316 uiHeight >>= 1; 1432 1317 Int trWidth = uiWidth, trHeight = uiHeight; 1433 Int trLastWidth = uiWidth << 1, trLastHeight = uiHeight << 1;1434 pcCU>getNSQTSize ( uiTrMode, uiAbsPartIdx, trWidth, trHeight );1435 pcCU>getNSQTSize ( uiTrMode  1, uiAbsPartIdx, trLastWidth, trLastHeight );1436 1318 UInt uiAddrOffset = trHeight * uiStride; 1437 1319 UInt uiCoefOffset = trWidth * trHeight; 1438 UInt uiPartOffset = pcCU>getTotalNumPart() >> ( uiTrMode << 1 ); 1439 UInt uiInterTUSplitDirection = pcCU>getInterTUSplitDirection ( trWidth, trHeight, trLastWidth, trLastHeight ); 1440 if( uiInterTUSplitDirection != 2 ) 1441 { 1442 invRecurTransformNxN( pcCU, uiAbsPartIdx, eTxt, rpcResidual, uiAddr , uiStride, uiWidth, uiHeight, uiMaxTrMode, uiTrMode, rpcCoeff ); rpcCoeff += uiCoefOffset; uiAbsPartIdx += uiPartOffset; 1443 invRecurTransformNxN( pcCU, uiAbsPartIdx, eTxt, rpcResidual, uiAddr + trWidth * uiInterTUSplitDirection + uiAddrOffset * ( 1  uiInterTUSplitDirection), uiStride, uiWidth, uiHeight, uiMaxTrMode, uiTrMode, rpcCoeff ); rpcCoeff += uiCoefOffset; uiAbsPartIdx += uiPartOffset; 1444 invRecurTransformNxN( pcCU, uiAbsPartIdx, eTxt, rpcResidual, uiAddr + 2 * trWidth * uiInterTUSplitDirection + 2 * uiAddrOffset * ( 1  uiInterTUSplitDirection), uiStride, uiWidth, uiHeight, uiMaxTrMode, uiTrMode, rpcCoeff ); rpcCoeff += uiCoefOffset; uiAbsPartIdx += uiPartOffset; 1445 invRecurTransformNxN( pcCU, uiAbsPartIdx, eTxt, rpcResidual, uiAddr + 3 * trWidth * uiInterTUSplitDirection + 3 * uiAddrOffset * ( 1  uiInterTUSplitDirection), uiStride, uiWidth, uiHeight, uiMaxTrMode, uiTrMode, rpcCoeff ); 1446 } 1447 else 1320 UInt uiPartOffset = pcCU>getTotalNumPart() >> ( uiTrMode << 1 ); 1448 1321 { 1449 1322 invRecurTransformNxN( pcCU, uiAbsPartIdx, eTxt, rpcResidual, uiAddr , uiStride, uiWidth, uiHeight, uiMaxTrMode, uiTrMode, rpcCoeff ); rpcCoeff += uiCoefOffset; uiAbsPartIdx += uiPartOffset; … … 1466 1339 * \param uiMode is Intra Prediction mode used in ModeDependent DCT/DST only 1467 1340 */ 1468 Void TComTrQuant::xT( UInt uiMode, Pel* piBlkResi, UInt uiStride, Int* psCoeff, Int iWidth, Int iHeight )1341 Void TComTrQuant::xT(Int bitDepth, UInt uiMode, Pel* piBlkResi, UInt uiStride, Int* psCoeff, Int iWidth, Int iHeight ) 1469 1342 { 1470 1343 #if MATRIX_MULT 1471 Int iSize = iWidth; 1472 if( iWidth != iHeight) 1473 { 1474 xTrMxN( piBlkResi, psCoeff, uiStride, (UInt)iWidth, (UInt)iHeight ); 1475 return; 1476 } 1477 xTr(piBlkResi,psCoeff,uiStride,(UInt)iSize,uiMode); 1344 Int iSize = iWidth; 1345 xTr(bitDepth, piBlkResi,psCoeff,uiStride,(UInt)iSize,uiMode); 1478 1346 #else 1479 1347 Int j; 1480 { 1481 short block[ 64 * 64 ]; 1482 short coeff[ 64 * 64 ]; 1483 { 1348 Short block[ 32 * 32 ]; 1349 Short coeff[ 32 * 32 ]; 1484 1350 for (j = 0; j < iHeight; j++) 1485 1351 { 1486 memcpy( block + j * iWidth, piBlkResi + j * uiStride, iWidth * sizeof( short ) ); 1487 } 1488 } 1489 xTrMxN( block, coeff, iWidth, iHeight, uiMode ); 1352 memcpy( block + j * iWidth, piBlkResi + j * uiStride, iWidth * sizeof( Short ) ); 1353 } 1354 xTrMxN(bitDepth, block, coeff, iWidth, iHeight, uiMode ); 1490 1355 for ( j = 0; j < iHeight * iWidth; j++ ) 1491 1356 { 1492 1357 psCoeff[ j ] = coeff[ j ]; 1493 1358 } 1494 return ;1495 }1496 1359 #endif 1497 1360 } 1361 1498 1362 1499 1363 /** Wrapper function between HM interface and core NxN inverse transform (2D) … … 1504 1368 * \param uiMode is Intra Prediction mode used in ModeDependent DCT/DST only 1505 1369 */ 1506 Void TComTrQuant::xIT( UInt uiMode, Int* plCoef, Pel* pResidual, UInt uiStride, Int iWidth, Int iHeight )1370 Void TComTrQuant::xIT(Int bitDepth, UInt uiMode, Int* plCoef, Pel* pResidual, UInt uiStride, Int iWidth, Int iHeight ) 1507 1371 { 1508 1372 #if MATRIX_MULT 1509 1373 Int iSize = iWidth; 1510 if( iWidth != iHeight ) 1511 { 1512 xITrMxN( plCoef, pResidual, uiStride, (UInt)iWidth, (UInt)iHeight ); 1513 return; 1514 } 1515 xITr(plCoef,pResidual,uiStride,(UInt)iSize,uiMode); 1374 xITr(bitDepth, plCoef,pResidual,uiStride,(UInt)iSize,uiMode); 1516 1375 #else 1517 1376 Int j; 1518 1377 { 1519 short block[ 64 * 64];1520 short coeff[ 64 * 64];1378 Short block[ 32 * 32 ]; 1379 Short coeff[ 32 * 32 ]; 1521 1380 for ( j = 0; j < iHeight * iWidth; j++ ) 1522 1381 { 1523 coeff[j] = ( short)plCoef[j];1524 } 1525 xITrMxN( coeff, block, iWidth, iHeight, uiMode );1382 coeff[j] = (Short)plCoef[j]; 1383 } 1384 xITrMxN(bitDepth, coeff, block, iWidth, iHeight, uiMode ); 1526 1385 { 1527 1386 for ( j = 0; j < iHeight; j++ ) 1528 1387 { 1529 memcpy( pResidual + j * uiStride, block + j * iWidth, iWidth * sizeof( short) );1388 memcpy( pResidual + j * uiStride, block + j * iWidth, iWidth * sizeof(Short) ); 1530 1389 } 1531 1390 } … … 1535 1394 } 1536 1395 1396 /** Wrapper function between HM interface and core 4x4 transform skipping 1397 * \param piBlkResi input data (residual) 1398 * \param psCoeff output data (transform coefficients) 1399 * \param uiStride stride of input residual data 1400 * \param iSize transform size (iSize x iSize) 1401 */ 1402 Void TComTrQuant::xTransformSkip(Int bitDepth, Pel* piBlkResi, UInt uiStride, Int* psCoeff, Int width, Int height ) 1403 { 1404 assert( width == height ); 1405 UInt uiLog2TrSize = g_aucConvertToBit[ width ] + 2; 1406 Int shift = MAX_TR_DYNAMIC_RANGE  bitDepth  uiLog2TrSize; 1407 UInt transformSkipShift; 1408 Int j,k; 1409 if(shift >= 0) 1410 { 1411 transformSkipShift = shift; 1412 for (j = 0; j < height; j++) 1413 { 1414 for(k = 0; k < width; k ++) 1415 { 1416 psCoeff[j*height + k] = piBlkResi[j * uiStride + k] << transformSkipShift; 1417 } 1418 } 1419 } 1420 else 1421 { 1422 //The case when uiBitDepth > 13 1423 Int offset; 1424 transformSkipShift = shift; 1425 offset = (1 << (transformSkipShift  1)); 1426 for (j = 0; j < height; j++) 1427 { 1428 for(k = 0; k < width; k ++) 1429 { 1430 psCoeff[j*height + k] = (piBlkResi[j * uiStride + k] + offset) >> transformSkipShift; 1431 } 1432 } 1433 } 1434 } 1435 1436 /** Wrapper function between HM interface and core NxN transform skipping 1437 * \param plCoef input data (coefficients) 1438 * \param pResidual output data (residual) 1439 * \param uiStride stride of input residual data 1440 * \param iSize transform size (iSize x iSize) 1441 */ 1442 Void TComTrQuant::xITransformSkip(Int bitDepth, Int* plCoef, Pel* pResidual, UInt uiStride, Int width, Int height ) 1443 { 1444 assert( width == height ); 1445 UInt uiLog2TrSize = g_aucConvertToBit[ width ] + 2; 1446 Int shift = MAX_TR_DYNAMIC_RANGE  bitDepth  uiLog2TrSize; 1447 UInt transformSkipShift; 1448 Int j,k; 1449 if(shift > 0) 1450 { 1451 Int offset; 1452 transformSkipShift = shift; 1453 offset = (1 << (transformSkipShift 1)); 1454 for ( j = 0; j < height; j++ ) 1455 { 1456 for(k = 0; k < width; k ++) 1457 { 1458 pResidual[j * uiStride + k] = (plCoef[j*width+k] + offset) >> transformSkipShift; 1459 } 1460 } 1461 } 1462 else 1463 { 1464 //The case when uiBitDepth >= 13 1465 transformSkipShift =  shift; 1466 for ( j = 0; j < height; j++ ) 1467 { 1468 for(k = 0; k < width; k ++) 1469 { 1470 pResidual[j * uiStride + k] = plCoef[j*width+k] << transformSkipShift; 1471 } 1472 } 1473 } 1474 } 1475 1537 1476 /** RDOQ with CABAC 1538 1477 * \param pcCU pointer to coding unit structure … … 1560 1499 UInt uiAbsPartIdx ) 1561 1500 { 1562 Int iQBits = m_cQP.m_iBits;1563 Double dTemp = 0;1564 1565 UInt dir = SCALING_LIST_SQT;1566 1501 UInt uiLog2TrSize = g_aucConvertToBit[ uiWidth ] + 2; 1567 Int uiQ = g_quantScales[m_cQP.rem()]; 1568 if (uiWidth != uiHeight) 1569 { 1570 uiLog2TrSize += (uiWidth > uiHeight) ? 1 : 1; 1571 dir = ( uiWidth < uiHeight )? SCALING_LIST_VER: SCALING_LIST_HOR; 1572 } 1573 1574 #if FULL_NBIT 1575 UInt uiBitDepth = g_uiBitDepth; 1576 #else 1577 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 1578 #endif 1502 1503 UInt uiBitDepth = eTType == TEXT_LUMA ? g_bitDepthY : g_bitDepthC; 1579 1504 Int iTransformShift = MAX_TR_DYNAMIC_RANGE  uiBitDepth  uiLog2TrSize; // Represents scaling through forward transform 1580 1505 UInt uiGoRiceParam = 0; … … 1584 1509 Int scalingListType = (pcCU>isIntra(uiAbsPartIdx) ? 0 : 3) + g_eTTable[(Int)eTType]; 1585 1510 assert(scalingListType < 6); 1586 1587 iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift; // Right shift of nonRDOQ quantizer; level = (coeff*uiQ + offset)>>q_bits 1588 double dErrScale = 0; 1589 double *pdErrScaleOrg = getErrScaleCoeff(scalingListType,uiLog2TrSize2,m_cQP.m_iRem,dir); 1590 Int *piQCoefOrg = getQuantCoeff(scalingListType,m_cQP.m_iRem,uiLog2TrSize2,dir); 1511 1512 Int iQBits = QUANT_SHIFT + m_cQP.m_iPer + iTransformShift; // Right shift of nonRDOQ quantizer; level = (coeff*uiQ + offset)>>q_bits 1513 Double *pdErrScaleOrg = getErrScaleCoeff(scalingListType,uiLog2TrSize2,m_cQP.m_iRem); 1514 Int *piQCoefOrg = getQuantCoeff(scalingListType,m_cQP.m_iRem,uiLog2TrSize2); 1591 1515 Int *piQCoef = piQCoefOrg; 1592 double *pdErrScale = pdErrScaleOrg;1516 Double *pdErrScale = pdErrScaleOrg; 1593 1517 #if ADAPTIVE_QP_SELECTION 1594 1518 Int iQBitsC = iQBits  ARL_C_PRECISION; … … 1596 1520 #endif 1597 1521 UInt uiScanIdx = pcCU>getCoefScanIdx(uiAbsPartIdx, uiWidth, eTType==TEXT_LUMA, pcCU>isIntra(uiAbsPartIdx)); 1598 if (uiScanIdx == SCAN_ZIGZAG)1599 {1600 // Map value zigzag to diagonal scan1601 uiScanIdx = SCAN_DIAG;1602 }1603 Int blockType = uiLog2BlkSize;1604 if (uiWidth != uiHeight)1605 {1606 uiScanIdx = SCAN_DIAG;1607 blockType = 4;1608 }1609 1522 1610 1523 #if ADAPTIVE_QP_SELECTION 1611 1524 memset(piArlDstCoeff, 0, sizeof(Int) * uiMaxNumCoeff); 1612 1525 #endif 1613 1526 1614 1527 Double pdCostCoeff [ 32 * 32 ]; 1615 1528 Double pdCostSig [ 32 * 32 ]; … … 1625 1538 ::memset( sigRateDelta, 0, sizeof(Int) * uiMaxNumCoeff ); 1626 1539 ::memset( deltaU, 0, sizeof(Int) * uiMaxNumCoeff ); 1627 1540 1628 1541 const UInt * scanCG; 1629 if (uiWidth == uiHeight)1630 1542 { 1631 1543 scanCG = g_auiSigLastScan[ uiScanIdx ][ uiLog2BlkSize > 3 ? uiLog2BlkSize21 : 0 ]; … … 1638 1550 scanCG = g_sigLastScanCG32x32; 1639 1551 } 1640 }1641 else1642 {1643 scanCG = g_sigCGScanNSQT[ uiLog2BlkSize  2 ];1644 1552 } 1645 1553 const UInt uiCGSize = (1 << MLS_CG_SIZE); // 16 … … 1648 1556 UInt uiNumBlkSide = uiWidth / MLS_CG_SIZE; 1649 1557 Int iCGLastScanPos = 1; 1650 1558 1651 1559 UInt uiCtxSet = 0; 1652 1560 Int c1 = 1; 1653 1561 Int c2 = 0; 1654 UInt uiNumOne = 0;1655 1562 Double d64BaseCost = 0; 1656 1563 Int iLastScanPos = 1; 1657 dTemp = dErrScale; 1658 1564 1659 1565 UInt c1Idx = 0; 1660 1566 UInt c2Idx = 0; 1661 1567 Int baseLevel; 1662 1663 const UInt * scan; 1664 if (uiWidth == uiHeight) 1665 { 1666 scan = g_auiSigLastScan[ uiScanIdx ][ uiLog2BlkSize  1 ]; 1667 } 1668 else 1669 { 1670 scan = g_sigScanNSQT[ uiLog2BlkSize  2 ]; 1671 } 1672 1673 ::memset( pdCostCoeffGroupSig, 0, sizeof(Double) * MLS_GRP_NUM ); 1674 ::memset( uiSigCoeffGroupFlag, 0, sizeof(UInt) * MLS_GRP_NUM ); 1675 1676 UInt uiCGNum = uiWidth * uiHeight >> MLS_CG_SIZE; 1677 Int iScanPos; 1678 coeffGroupRDStats rdStats; 1679 1680 for (Int iCGScanPos = uiCGNum1; iCGScanPos >= 0; iCGScanPos) 1681 { 1682 UInt uiCGBlkPos = scanCG[ iCGScanPos ]; 1683 UInt uiCGPosY = uiCGBlkPos / uiNumBlkSide; 1684 UInt uiCGPosX = uiCGBlkPos  (uiCGPosY * uiNumBlkSide); 1685 if( uiWidth == 8 && uiHeight == 8 && (uiScanIdx == SCAN_HOR  uiScanIdx == SCAN_VER) ) 1686 { 1687 uiCGPosY = (uiScanIdx == SCAN_HOR ? uiCGBlkPos : 0); 1688 uiCGPosX = (uiScanIdx == SCAN_VER ? uiCGBlkPos : 0); 1689 } 1690 ::memset( &rdStats, 0, sizeof (coeffGroupRDStats)); 1568 1569 const UInt *scan = g_auiSigLastScan[ uiScanIdx ][ uiLog2BlkSize  1 ]; 1570 1571 ::memset( pdCostCoeffGroupSig, 0, sizeof(Double) * MLS_GRP_NUM ); 1572 ::memset( uiSigCoeffGroupFlag, 0, sizeof(UInt) * MLS_GRP_NUM ); 1573 1574 UInt uiCGNum = uiWidth * uiHeight >> MLS_CG_SIZE; 1575 Int iScanPos; 1576 coeffGroupRDStats rdStats; 1577 1578 for (Int iCGScanPos = uiCGNum1; iCGScanPos >= 0; iCGScanPos) 1579 { 1580 UInt uiCGBlkPos = scanCG[ iCGScanPos ]; 1581 UInt uiCGPosY = uiCGBlkPos / uiNumBlkSide; 1582 UInt uiCGPosX = uiCGBlkPos  (uiCGPosY * uiNumBlkSide); 1583 ::memset( &rdStats, 0, sizeof (coeffGroupRDStats)); 1584 1585 const Int patternSigCtx = TComTrQuant::calcPatternSigCtx(uiSigCoeffGroupFlag, uiCGPosX, uiCGPosY, uiWidth, uiHeight); 1586 for (Int iScanPosinCG = uiCGSize1; iScanPosinCG >= 0; iScanPosinCG) 1587 { 1588 iScanPos = iCGScanPos*uiCGSize + iScanPosinCG; 1589 //===== quantization ===== 1590 UInt uiBlkPos = scan[iScanPos]; 1591 // set coeff 1592 Int uiQ = piQCoef[uiBlkPos]; 1593 Double dTemp = pdErrScale[uiBlkPos]; 1594 Int lLevelDouble = plSrcCoeff[ uiBlkPos ]; 1595 lLevelDouble = (Int)min<Int64>((Int64)abs((Int)lLevelDouble) * uiQ , MAX_INT  (1 << (iQBits  1))); 1596 #if ADAPTIVE_QP_SELECTION 1597 if( m_bUseAdaptQpSelect ) 1598 { 1599 piArlDstCoeff[uiBlkPos] = (Int)(( lLevelDouble + iAddC) >> iQBitsC ); 1600 } 1601 #endif 1602 UInt uiMaxAbsLevel = (lLevelDouble + (1 << (iQBits  1))) >> iQBits; 1603 1604 Double dErr = Double( lLevelDouble ); 1605 pdCostCoeff0[ iScanPos ] = dErr * dErr * dTemp; 1606 d64BlockUncodedCost += pdCostCoeff0[ iScanPos ]; 1607 piDstCoeff[ uiBlkPos ] = uiMaxAbsLevel; 1608 1609 if ( uiMaxAbsLevel > 0 && iLastScanPos < 0 ) 1610 { 1611 iLastScanPos = iScanPos; 1612 uiCtxSet = (iScanPos < SCAN_SET_SIZE  eTType!=TEXT_LUMA) ? 0 : 2; 1613 iCGLastScanPos = iCGScanPos; 1614 } 1615 1616 if ( iLastScanPos >= 0 ) 1617 { 1618 //===== coefficient level estimation ===== 1619 UInt uiLevel; 1620 UInt uiOneCtx = 4 * uiCtxSet + c1; 1621 UInt uiAbsCtx = uiCtxSet + c2; 1691 1622 1692 for (Int iScanPosinCG = uiCGSize1; iScanPosinCG >= 0; iScanPosinCG) 1693 { 1694 iScanPos = iCGScanPos*uiCGSize + iScanPosinCG; 1695 //===== quantization ===== 1696 UInt uiBlkPos = scan[iScanPos]; 1697 // set coeff 1698 uiQ = piQCoef[uiBlkPos]; 1699 dTemp = pdErrScale[uiBlkPos]; 1700 Int lLevelDouble = plSrcCoeff[ uiBlkPos ]; 1701 lLevelDouble = (Int)min<Int64>((Int64)abs((Int)lLevelDouble) * uiQ , MAX_INT  (1 << (iQBits  1))); 1702 #if ADAPTIVE_QP_SELECTION 1703 if( m_bUseAdaptQpSelect ) 1623 if( iScanPos == iLastScanPos ) 1704 1624 { 1705 piArlDstCoeff[uiBlkPos] = (Int)(( lLevelDouble + iAddC) >> iQBitsC ); 1706 } 1707 #endif 1708 UInt uiMaxAbsLevel = (lLevelDouble + (1 << (iQBits  1))) >> iQBits; 1709 1710 Double dErr = Double( lLevelDouble ); 1711 pdCostCoeff0[ iScanPos ] = dErr * dErr * dTemp; 1712 d64BlockUncodedCost += pdCostCoeff0[ iScanPos ]; 1713 piDstCoeff[ uiBlkPos ] = uiMaxAbsLevel; 1714 1715 if ( uiMaxAbsLevel > 0 && iLastScanPos < 0 ) 1716 { 1717 iLastScanPos = iScanPos; 1718 uiCtxSet = (iScanPos < SCAN_SET_SIZE  eTType!=TEXT_LUMA) ? 0 : 2; 1719 iCGLastScanPos = iCGScanPos; 1720 } 1721 1722 if ( iLastScanPos >= 0 ) 1723 { 1724 //===== coefficient level estimation ===== 1725 UInt uiLevel; 1726 UInt uiOneCtx = 4 * uiCtxSet + c1; 1727 UInt uiAbsCtx = uiCtxSet + c2; 1728 1729 if( iScanPos == iLastScanPos ) 1730 { 1731 uiLevel = xGetCodedLevel( pdCostCoeff[ iScanPos ], pdCostCoeff0[ iScanPos ], pdCostSig[ iScanPos ], 1732 lLevelDouble, uiMaxAbsLevel, 0, uiOneCtx, uiAbsCtx, uiGoRiceParam, 1733 c1Idx, c2Idx, iQBits, dTemp, 1 ); 1734 } 1735 else 1736 { 1737 UInt uiPosY = uiBlkPos >> uiLog2BlkSize; 1738 UInt uiPosX = uiBlkPos  ( uiPosY << uiLog2BlkSize ); 1739 UShort uiCtxSig = getSigCtxInc( piDstCoeff, uiPosX, uiPosY, blockType, uiWidth, uiHeight, eTType ); 1740 uiLevel = xGetCodedLevel( pdCostCoeff[ iScanPos ], pdCostCoeff0[ iScanPos ], pdCostSig[ iScanPos ], 1741 lLevelDouble, uiMaxAbsLevel, uiCtxSig, uiOneCtx, uiAbsCtx, uiGoRiceParam, 1742 c1Idx, c2Idx, iQBits, dTemp, 0 ); 1743 sigRateDelta[ uiBlkPos ] = m_pcEstBitsSbac>significantBits[ uiCtxSig ][ 1 ]  m_pcEstBitsSbac>significantBits[ uiCtxSig ][ 0 ]; 1744 } 1745 deltaU[ uiBlkPos ] = (lLevelDouble  ((Int)uiLevel << iQBits)) >> (iQBits8); 1746 if( uiLevel > 0 ) 1747 { 1748 Int rateNow = xGetICRate( uiLevel, uiOneCtx, uiAbsCtx, uiGoRiceParam, c1Idx, c2Idx ); 1749 rateIncUp [ uiBlkPos ] = xGetICRate( uiLevel+1, uiOneCtx, uiAbsCtx, uiGoRiceParam, c1Idx, c2Idx )  rateNow; 1750 rateIncDown [ uiBlkPos ] = xGetICRate( uiLevel1, uiOneCtx, uiAbsCtx, uiGoRiceParam, c1Idx, c2Idx )  rateNow; 1751 } 1752 else // uiLevel == 0 1753 { 1754 rateIncUp [ uiBlkPos ] = m_pcEstBitsSbac>m_greaterOneBits[ uiOneCtx ][ 0 ]; 1755 } 1756 piDstCoeff[ uiBlkPos ] = uiLevel; 1757 d64BaseCost += pdCostCoeff [ iScanPos ]; 1758 1759 1760 baseLevel = (c1Idx < C1FLAG_NUMBER) ? (2 + (c2Idx < C2FLAG_NUMBER)) : 1; 1761 if( uiLevel >= baseLevel ) 1762 { 1763 uiGoRiceParam = g_aauiGoRiceUpdate[ uiGoRiceParam ][ min<UInt>( uiLevel  baseLevel , 23 ) ]; 1764 } 1765 if ( uiLevel >= 1) 1766 { 1767 c1Idx ++; 1768 } 1769 1770 //===== update bin model ===== 1771 if( uiLevel > 1 ) 1772 { 1773 c1 = 0; 1774 c2 += (c2 < 2); 1775 uiNumOne++; 1776 c2Idx ++; 1777 } 1778 else if( (c1 < 3) && (c1 > 0) && uiLevel) 1779 { 1780 c1++; 1781 } 1782 1783 //===== context set update ===== 1784 if( ( iScanPos % SCAN_SET_SIZE == 0 ) && ( iScanPos > 0 ) ) 1785 { 1786 c1 = 1; 1787 c2 = 0; 1788 uiGoRiceParam = 0; 1789 1790 c1Idx = 0; 1791 c2Idx = 0; 1792 uiCtxSet = (iScanPos == SCAN_SET_SIZE  eTType!=TEXT_LUMA) ? 0 : 2; 1793 if( uiNumOne > 0 ) 1794 { 1795 uiCtxSet++; 1796 } 1797 uiNumOne >>= 1; 1798 } 1625 uiLevel = xGetCodedLevel( pdCostCoeff[ iScanPos ], pdCostCoeff0[ iScanPos ], pdCostSig[ iScanPos ], 1626 lLevelDouble, uiMaxAbsLevel, 0, uiOneCtx, uiAbsCtx, uiGoRiceParam, 1627 c1Idx, c2Idx, iQBits, dTemp, 1 ); 1799 1628 } 1800 1629 else 1801 1630 { 1802 d64BaseCost += pdCostCoeff0[ iScanPos ]; 1631 UInt uiPosY = uiBlkPos >> uiLog2BlkSize; 1632 UInt uiPosX = uiBlkPos  ( uiPosY << uiLog2BlkSize ); 1633 UShort uiCtxSig = getSigCtxInc( patternSigCtx, uiScanIdx, uiPosX, uiPosY, uiLog2BlkSize, eTType ); 1634 uiLevel = xGetCodedLevel( pdCostCoeff[ iScanPos ], pdCostCoeff0[ iScanPos ], pdCostSig[ iScanPos ], 1635 lLevelDouble, uiMaxAbsLevel, uiCtxSig, uiOneCtx, uiAbsCtx, uiGoRiceParam, 1636 c1Idx, c2Idx, iQBits, dTemp, 0 ); 1637 sigRateDelta[ uiBlkPos ] = m_pcEstBitsSbac>significantBits[ uiCtxSig ][ 1 ]  m_pcEstBitsSbac>significantBits[ uiCtxSig ][ 0 ]; 1803 1638 } 1804 rdStats.d64SigCost += pdCostSig[ iScanPos ];1805 if (iScanPosinCG ==0 )1639 deltaU[ uiBlkPos ] = (lLevelDouble  ((Int)uiLevel << iQBits)) >> (iQBits8); 1640 if( uiLevel > 0 ) 1806 1641 { 1807 rdStats.d64SigCost_0 = pdCostSig[ iScanPos ]; 1642 Int rateNow = xGetICRate( uiLevel, uiOneCtx, uiAbsCtx, uiGoRiceParam, c1Idx, c2Idx ); 1643 rateIncUp [ uiBlkPos ] = xGetICRate( uiLevel+1, uiOneCtx, uiAbsCtx, uiGoRiceParam, c1Idx, c2Idx )  rateNow; 1644 rateIncDown [ uiBlkPos ] = xGetICRate( uiLevel1, uiOneCtx, uiAbsCtx, uiGoRiceParam, c1Idx, c2Idx )  rateNow; 1808 1645 } 1809 if (piDstCoeff[ uiBlkPos ] )1646 else // uiLevel == 0 1810 1647 { 1811 uiSigCoeffGroupFlag[ uiCGBlkPos ] = 1; 1812 rdStats.d64CodedLevelandDist += pdCostCoeff[ iScanPos ]  pdCostSig[ iScanPos ]; 1813 rdStats.d64UncodedDist += pdCostCoeff0[ iScanPos ]; 1814 if ( iScanPosinCG != 0 ) 1648 rateIncUp [ uiBlkPos ] = m_pcEstBitsSbac>m_greaterOneBits[ uiOneCtx ][ 0 ]; 1649 } 1650 piDstCoeff[ uiBlkPos ] = uiLevel; 1651 d64BaseCost += pdCostCoeff [ iScanPos ]; 1652 1653 1654 baseLevel = (c1Idx < C1FLAG_NUMBER) ? (2 + (c2Idx < C2FLAG_NUMBER)) : 1; 1655 if( uiLevel >= baseLevel ) 1656 { 1657 if(uiLevel > 3*(1<<uiGoRiceParam)) 1815 1658 { 1816 rdStats.iNNZbeforePos0++;1659 uiGoRiceParam = min<UInt>(uiGoRiceParam+ 1, 4); 1817 1660 } 1818 1661 } 1819 } //end for (iScanPosinCG) 1820 1821 if (iCGLastScanPos >= 0) 1822 { 1823 if( iCGScanPos ) 1662 if ( uiLevel >= 1) 1824 1663 { 1825 if (uiSigCoeffGroupFlag[ uiCGBlkPos ] == 0) 1664 c1Idx ++; 1665 } 1666 1667 //===== update bin model ===== 1668 if( uiLevel > 1 ) 1669 { 1670 c1 = 0; 1671 c2 += (c2 < 2); 1672 c2Idx ++; 1673 } 1674 else if( (c1 < 3) && (c1 > 0) && uiLevel) 1675 { 1676 c1++; 1677 } 1678 1679 //===== context set update ===== 1680 if( ( iScanPos % SCAN_SET_SIZE == 0 ) && ( iScanPos > 0 ) ) 1681 { 1682 c2 = 0; 1683 uiGoRiceParam = 0; 1684 1685 c1Idx = 0; 1686 c2Idx = 0; 1687 uiCtxSet = (iScanPos == SCAN_SET_SIZE  eTType!=TEXT_LUMA) ? 0 : 2; 1688 if( c1 == 0 ) 1826 1689 { 1827 UInt uiCtxSig = getSigCoeffGroupCtxInc( uiSigCoeffGroupFlag, uiCGPosX, uiCGPosY, uiScanIdx, uiWidth, uiHeight); 1828 d64BaseCost += xGetRateSigCoeffGroup(0, uiCtxSig)  rdStats.d64SigCost;; 1829 pdCostCoeffGroupSig[ iCGScanPos ] = xGetRateSigCoeffGroup(0, uiCtxSig); 1830 } 1831 else 1690 uiCtxSet++; 1691 } 1692 c1 = 1; 1693 } 1694 } 1695 else 1696 { 1697 d64BaseCost += pdCostCoeff0[ iScanPos ]; 1698 } 1699 rdStats.d64SigCost += pdCostSig[ iScanPos ]; 1700 if (iScanPosinCG == 0 ) 1701 { 1702 rdStats.d64SigCost_0 = pdCostSig[ iScanPos ]; 1703 } 1704 if (piDstCoeff[ uiBlkPos ] ) 1705 { 1706 uiSigCoeffGroupFlag[ uiCGBlkPos ] = 1; 1707 rdStats.d64CodedLevelandDist += pdCostCoeff[ iScanPos ]  pdCostSig[ iScanPos ]; 1708 rdStats.d64UncodedDist += pdCostCoeff0[ iScanPos ]; 1709 if ( iScanPosinCG != 0 ) 1710 { 1711 rdStats.iNNZbeforePos0++; 1712 } 1713 } 1714 } //end for (iScanPosinCG) 1715 1716 if (iCGLastScanPos >= 0) 1717 { 1718 if( iCGScanPos ) 1719 { 1720 if (uiSigCoeffGroupFlag[ uiCGBlkPos ] == 0) 1721 { 1722 UInt uiCtxSig = getSigCoeffGroupCtxInc( uiSigCoeffGroupFlag, uiCGPosX, uiCGPosY, uiWidth, uiHeight); 1723 d64BaseCost += xGetRateSigCoeffGroup(0, uiCtxSig)  rdStats.d64SigCost;; 1724 pdCostCoeffGroupSig[ iCGScanPos ] = xGetRateSigCoeffGroup(0, uiCtxSig); 1725 } 1726 else 1727 { 1728 if (iCGScanPos < iCGLastScanPos) //skip the last coefficient group, which will be handled together with last position below. 1832 1729 { 1833 if ( iCGScanPos < iCGLastScanPos) //skip the last coefficient group, which will be handled together with last position below.1730 if ( rdStats.iNNZbeforePos0 == 0 ) 1834 1731 { 1835 if ( rdStats.iNNZbeforePos0 == 0 ) 1836 { 1837 d64BaseCost = rdStats.d64SigCost_0; 1838 rdStats.d64SigCost = rdStats.d64SigCost_0; 1839 } 1840 // rdcost if SigCoeffGroupFlag = 0, initialization 1841 Double d64CostZeroCG = d64BaseCost; 1842 1843 // add SigCoeffGroupFlag cost to total cost 1844 UInt uiCtxSig = getSigCoeffGroupCtxInc( uiSigCoeffGroupFlag, uiCGPosX, uiCGPosY, uiScanIdx, uiWidth, uiHeight); 1732 d64BaseCost = rdStats.d64SigCost_0; 1733 rdStats.d64SigCost = rdStats.d64SigCost_0; 1734 } 1735 // rdcost if SigCoeffGroupFlag = 0, initialization 1736 Double d64CostZeroCG = d64BaseCost; 1737 1738 // add SigCoeffGroupFlag cost to total cost 1739 UInt uiCtxSig = getSigCoeffGroupCtxInc( uiSigCoeffGroupFlag, uiCGPosX, uiCGPosY, uiWidth, uiHeight); 1740 if (iCGScanPos < iCGLastScanPos) 1741 { 1742 d64BaseCost += xGetRateSigCoeffGroup(1, uiCtxSig); 1743 d64CostZeroCG += xGetRateSigCoeffGroup(0, uiCtxSig); 1744 pdCostCoeffGroupSig[ iCGScanPos ] = xGetRateSigCoeffGroup(1, uiCtxSig); 1745 } 1746 1747 // try to convert the current coeff group from nonzero to allzero 1748 d64CostZeroCG += rdStats.d64UncodedDist; // distortion for resetting nonzero levels to zero levels 1749 d64CostZeroCG = rdStats.d64CodedLevelandDist; // distortion and level cost for keeping all nonzero levels 1750 d64CostZeroCG = rdStats.d64SigCost; // sig cost for all coeffs, including zero levels and nonzerl levels 1751 1752 // if we can save cost, change this block to allzero block 1753 if ( d64CostZeroCG < d64BaseCost ) 1754 { 1755 uiSigCoeffGroupFlag[ uiCGBlkPos ] = 0; 1756 d64BaseCost = d64CostZeroCG; 1845 1757 if (iCGScanPos < iCGLastScanPos) 1846 1758 { 1847 d64BaseCost += xGetRateSigCoeffGroup(1, uiCtxSig); 1848 d64CostZeroCG += xGetRateSigCoeffGroup(0, uiCtxSig); 1849 pdCostCoeffGroupSig[ iCGScanPos ] = xGetRateSigCoeffGroup(1, uiCtxSig); 1759 pdCostCoeffGroupSig[ iCGScanPos ] = xGetRateSigCoeffGroup(0, uiCtxSig); 1850 1760 } 1851 1852 // try to convert the current coeff group from nonzero to allzero 1853 d64CostZeroCG += rdStats.d64UncodedDist; // distortion for resetting nonzero levels to zero levels 1854 d64CostZeroCG = rdStats.d64CodedLevelandDist; // distortion and level cost for keeping all nonzero levels 1855 d64CostZeroCG = rdStats.d64SigCost; // sig cost for all coeffs, including zero levels and nonzerl levels 1856 1857 // if we can save cost, change this block to allzero block 1858 if ( d64CostZeroCG < d64BaseCost ) 1761 // reset coeffs to 0 in this block 1762 for (Int iScanPosinCG = uiCGSize1; iScanPosinCG >= 0; iScanPosinCG) 1859 1763 { 1860 uiSigCoeffGroupFlag[ uiCGBlkPos ] = 0; 1861 d64BaseCost = d64CostZeroCG; 1862 if (iCGScanPos < iCGLastScanPos) 1764 iScanPos = iCGScanPos*uiCGSize + iScanPosinCG; 1765 UInt uiBlkPos = scan[ iScanPos ]; 1766 1767 if (piDstCoeff[ uiBlkPos ]) 1863 1768 { 1864 pdCostCoeffGroupSig[ iCGScanPos ] = xGetRateSigCoeffGroup(0, uiCtxSig); 1769 piDstCoeff [ uiBlkPos ] = 0; 1770 pdCostCoeff[ iScanPos ] = pdCostCoeff0[ iScanPos ]; 1771 pdCostSig [ iScanPos ] = 0; 1865 1772 } 1866 // reset coeffs to 0 in this block 1867 for (Int iScanPosinCG = uiCGSize1; iScanPosinCG >= 0; iScanPosinCG) 1868 { 1869 iScanPos = iCGScanPos*uiCGSize + iScanPosinCG; 1870 UInt uiBlkPos = scan[ iScanPos ]; 1871 1872 if (piDstCoeff[ uiBlkPos ]) 1873 { 1874 piDstCoeff [ uiBlkPos ] = 0; 1875 pdCostCoeff[ iScanPos ] = pdCostCoeff0[ iScanPos ]; 1876 pdCostSig [ iScanPos ] = 0; 1877 } 1878 } 1879 } // end if ( d64CostAllZeros < d64BaseCost ) 1880 } 1881 } // end if if (uiSigCoeffGroupFlag[ uiCGBlkPos ] == 0) 1882 } 1883 else 1884 { 1885 uiSigCoeffGroupFlag[ uiCGBlkPos ] = 1; 1886 } 1887 } 1888 } //end for (iCGScanPos) 1889 1773 } 1774 } // end if ( d64CostAllZeros < d64BaseCost ) 1775 } 1776 } // end if if (uiSigCoeffGroupFlag[ uiCGBlkPos ] == 0) 1777 } 1778 else 1779 { 1780 uiSigCoeffGroupFlag[ uiCGBlkPos ] = 1; 1781 } 1782 } 1783 } //end for (iCGScanPos) 1784 1890 1785 //===== estimate last position ===== 1891 1786 if ( iLastScanPos < 0 ) … … 1893 1788 return; 1894 1789 } 1895 1790 1896 1791 Double d64BestCost = 0; 1897 1792 Int ui16CtxCbf = 0; … … 1905 1800 else 1906 1801 { 1907 ui16CtxCbf = pcCU>getCtxQtCbf( uiAbsPartIdx,eTType, pcCU>getTransformIdx( uiAbsPartIdx ) );1802 ui16CtxCbf = pcCU>getCtxQtCbf( eTType, pcCU>getTransformIdx( uiAbsPartIdx ) ); 1908 1803 ui16CtxCbf = ( eTType ? TEXT_CHROMA : eTType ) * NUM_QT_CBF_CTX + ui16CtxCbf; 1909 1804 d64BestCost = d64BlockUncodedCost + xGetICost( m_pcEstBitsSbac>blockCbpBits[ ui16CtxCbf ][ 0 ] ); 1910 1805 d64BaseCost += xGetICost( m_pcEstBitsSbac>blockCbpBits[ ui16CtxCbf ][ 1 ] ); 1911 1806 } 1912 1913 Bool bFoundLast = false; 1914 for (Int iCGScanPos = iCGLastScanPos; iCGScanPos >= 0; iCGScanPos) 1915 { 1916 UInt uiCGBlkPos = scanCG[ iCGScanPos ]; 1917 1918 d64BaseCost = pdCostCoeffGroupSig [ iCGScanPos ]; 1919 if (uiSigCoeffGroupFlag[ uiCGBlkPos ]) 1920 { 1921 for (Int iScanPosinCG = uiCGSize1; iScanPosinCG >= 0; iScanPosinCG) 1807 1808 Bool bFoundLast = false; 1809 for (Int iCGScanPos = iCGLastScanPos; iCGScanPos >= 0; iCGScanPos) 1810 { 1811 UInt uiCGBlkPos = scanCG[ iCGScanPos ]; 1812 1813 d64BaseCost = pdCostCoeffGroupSig [ iCGScanPos ]; 1814 if (uiSigCoeffGroupFlag[ uiCGBlkPos ]) 1815 { 1816 for (Int iScanPosinCG = uiCGSize1; iScanPosinCG >= 0; iScanPosinCG) 1817 { 1818 iScanPos = iCGScanPos*uiCGSize + iScanPosinCG; 1819 if (iScanPos > iLastScanPos) continue; 1820 UInt uiBlkPos = scan[iScanPos]; 1821 1822 if( piDstCoeff[ uiBlkPos ] ) 1922 1823 { 1923 iScanPos = iCGScanPos*uiCGSize + iScanPosinCG; 1924 if (iScanPos > iLastScanPos) continue; 1925 UInt uiBlkPos = scan[iScanPos]; 1926 1927 if( piDstCoeff[ uiBlkPos ] ) 1824 UInt uiPosY = uiBlkPos >> uiLog2BlkSize; 1825 UInt uiPosX = uiBlkPos  ( uiPosY << uiLog2BlkSize ); 1826 1827 Double d64CostLast= uiScanIdx == SCAN_VER ? xGetRateLast( uiPosY, uiPosX ) : xGetRateLast( uiPosX, uiPosY ); 1828 Double totalCost = d64BaseCost + d64CostLast  pdCostSig[ iScanPos ]; 1829 1830 if( totalCost < d64BestCost ) 1928 1831 { 1929 UInt uiPosY = uiBlkPos >> uiLog2BlkSize; 1930 UInt uiPosX = uiBlkPos  ( uiPosY << uiLog2BlkSize ); 1931 1932 Double d64CostLast= uiScanIdx == SCAN_VER ? xGetRateLast( uiPosY, uiPosX, uiWidth ) : xGetRateLast( uiPosX, uiPosY, uiWidth ); 1933 Double totalCost = d64BaseCost + d64CostLast  pdCostSig[ iScanPos ]; 1934 1935 if( totalCost < d64BestCost ) 1936 { 1937 iBestLastIdxP1 = iScanPos + 1; 1938 d64BestCost = totalCost; 1939 } 1940 if( piDstCoeff[ uiBlkPos ] > 1 ) 1941 { 1942 bFoundLast = true; 1943 break; 1944 } 1945 d64BaseCost = pdCostCoeff[ iScanPos ]; 1946 d64BaseCost += pdCostCoeff0[ iScanPos ]; 1832 iBestLastIdxP1 = iScanPos + 1; 1833 d64BestCost = totalCost; 1947 1834 } 1948 else1835 if( piDstCoeff[ uiBlkPos ] > 1 ) 1949 1836 { 1950 d64BaseCost = pdCostSig[ iScanPos ]; 1837 bFoundLast = true; 1838 break; 1951 1839 } 1952 } //end for 1953 if (bFoundLast) 1840 d64BaseCost = pdCostCoeff[ iScanPos ]; 1841 d64BaseCost += pdCostCoeff0[ iScanPos ]; 1842 } 1843 else 1954 1844 { 1955 break;1845 d64BaseCost = pdCostSig[ iScanPos ]; 1956 1846 } 1957 } // end if (uiSigCoeffGroupFlag[ uiCGBlkPos ]) 1958 } // end for 1959 1847 } //end for 1848 if (bFoundLast) 1849 { 1850 break; 1851 } 1852 } // end if (uiSigCoeffGroupFlag[ uiCGBlkPos ]) 1853 } // end for 1854 1960 1855 for ( Int scanPos = 0; scanPos < iBestLastIdxP1; scanPos++ ) 1961 1856 { … … 1971 1866 piDstCoeff[ scan[ scanPos ] ] = 0; 1972 1867 } 1973 1868 1974 1869 if( pcCU>getSlice()>getPPS()>getSignHideFlag() && uiAbsSum>=2) 1975 1870 { 1976 Int rdFactor = (Int)((Double)(g_invQuantScales[m_cQP.rem()]*g_invQuantScales[m_cQP.rem()]<<(2*m_cQP.m_iPer))/m_dLambda/16 + 0.5) ;1977 1978 Int tsig = pcCU>getSlice()>getPPS()>getTSIG() ;1979 1871 Int64 rdFactor = (Int64) ( 1872 g_invQuantScales[m_cQP.rem()] * g_invQuantScales[m_cQP.rem()] * (1<<(2*m_cQP.m_iPer)) 1873 / m_dLambda / 16 / (1<<DISTORTION_PRECISION_ADJUSTMENT(2*(uiBitDepth8))) 1874 + 0.5); 1980 1875 Int lastCG = 1; 1981 1876 Int absSum = 0 ; 1982 1877 Int n ; 1983 1878 1984 1879 for( Int subSet = (uiWidth*uiHeight1) >> LOG2_SCAN_SET_SIZE; subSet >= 0; subSet ) 1985 1880 { … … 1987 1882 Int firstNZPosInCG=SCAN_SET_SIZE , lastNZPosInCG=1 ; 1988 1883 absSum = 0 ; 1989 1884 1990 1885 for(n = SCAN_SET_SIZE1; n >= 0; n ) 1991 1886 { … … 1996 1891 } 1997 1892 } 1998 1893 1999 1894 for(n = 0; n <SCAN_SET_SIZE; n++ ) 2000 1895 { … … 2005 1900 } 2006 1901 } 2007 1902 2008 1903 for(n = firstNZPosInCG; n <=lastNZPosInCG; n++ ) 2009 1904 { 2010 1905 absSum += piDstCoeff[ scan[ n + subPos ]]; 2011 1906 } 2012 2013 if(lastNZPosInCG>=0 && lastCG==1) lastCG =1 ;2014 1907 2015 if( lastNZPosInCGfirstNZPosInCG>=tsig ) 1908 if(lastNZPosInCG>=0 && lastCG==1) 1909 { 1910 lastCG = 1; 1911 } 1912 1913 if( lastNZPosInCGfirstNZPosInCG>=SBH_THRESHOLD ) 2016 1914 { 2017 1915 UInt signbit = (piDstCoeff[scan[subPos+firstNZPosInCG]]>0?0:1); … … 2019 1917 { 2020 1918 // calculate the cost 2021 Int minCostInc = MAX_INT, minPos =1, finalChange=0, curCost=MAX_INT, curChange=0; 2022 1919 Int64 minCostInc = MAX_INT64, curCost=MAX_INT64; 1920 Int minPos =1, finalChange=0, curChange=0; 1921 2023 1922 for( n = (lastCG==1?lastNZPosInCG:SCAN_SET_SIZE1) ; n >= 0; n ) 2024 1923 { … … 2026 1925 if(piDstCoeff[ uiBlkPos ] != 0 ) 2027 1926 { 2028 Int costUp = rdFactor * (  deltaU[uiBlkPos] ) + rateIncUp[uiBlkPos] ;2029 Int costDown = rdFactor * ( deltaU[uiBlkPos] ) + rateIncDown[uiBlkPos]2030 2031 1927 Int64 costUp = rdFactor * (  deltaU[uiBlkPos] ) + rateIncUp[uiBlkPos] ; 1928 Int64 costDown = rdFactor * ( deltaU[uiBlkPos] ) + rateIncDown[uiBlkPos] 1929  ( abs(piDstCoeff[uiBlkPos])==1?((1<<15)+sigRateDelta[uiBlkPos]):0 ); 1930 2032 1931 if(lastCG==1 && lastNZPosInCG==n && abs(piDstCoeff[uiBlkPos])==1) 2033 1932 { 2034 1933 costDown = (4<<15) ; 2035 1934 } 2036 1935 2037 1936 if(costUp<costDown) 2038 1937 { … … 2045 1944 if(n==firstNZPosInCG && abs(piDstCoeff[uiBlkPos])==1) 2046 1945 { 2047 curCost = MAX_INT ;1946 curCost = MAX_INT64 ; 2048 1947 } 2049 1948 else … … 2057 1956 curCost = rdFactor * (  (abs(deltaU[uiBlkPos])) ) + (1<<15) + rateIncUp[uiBlkPos] + sigRateDelta[uiBlkPos] ; 2058 1957 curChange = 1 ; 2059 1958 2060 1959 if(n<firstNZPosInCG) 2061 1960 { … … 2063 1962 if(thissignbit != signbit ) 2064 1963 { 2065 curCost = MAX_INT ;1964 curCost = MAX_INT64; 2066 1965 } 2067 1966 } 2068 1967 } 2069 1968 2070 1969 if( curCost<minCostInc) 2071 1970 { … … 2075 1974 } 2076 1975 } 2077 2078 if(pi QCoef[minPos] == 32767  piQCoef[minPos] == 32768)1976 1977 if(piDstCoeff[minPos] == 32767  piDstCoeff[minPos] == 32768) 2079 1978 { 2080 1979 finalChange = 1; 2081 1980 } 2082 1981 2083 1982 if(plSrcCoeff[minPos]>=0) 2084 1983 { … … 2091 1990 } 2092 1991 } 2093 1992 2094 1993 if(lastCG==1) 2095 1994 { … … 2100 1999 } 2101 2000 2001 /** Pattern decision for context derivation process of significant_coeff_flag 2002 * \param sigCoeffGroupFlag pointer to prior coded significant coeff group 2003 * \param posXCG column of current coefficient group 2004 * \param posYCG row of current coefficient group 2005 * \param width width of the block 2006 * \param height height of the block 2007 * \returns pattern for current coefficient group 2008 */ 2009 Int TComTrQuant::calcPatternSigCtx( const UInt* sigCoeffGroupFlag, UInt posXCG, UInt posYCG, Int width, Int height ) 2010 { 2011 if( width == 4 && height == 4 ) return 1; 2012 2013 UInt sigRight = 0; 2014 UInt sigLower = 0; 2015 2016 width >>= 2; 2017 height >>= 2; 2018 if( posXCG < width  1 ) 2019 { 2020 sigRight = (sigCoeffGroupFlag[ posYCG * width + posXCG + 1 ] != 0); 2021 } 2022 if (posYCG < height  1 ) 2023 { 2024 sigLower = (sigCoeffGroupFlag[ (posYCG + 1 ) * width + posXCG ] != 0); 2025 } 2026 return sigRight + (sigLower<<1); 2027 } 2028 2102 2029 /** Context derivation process of coeff_abs_significant_flag 2103 * \param p cCoeff pointer to prior coded transform coefficients2030 * \param patternSigCtx pattern for current coefficient group 2104 2031 * \param posX column of current scan position 2105 2032 * \param posY row of current scan position 2106 * \param blockType log2 value of block size if square block, or 4 otherwise2033 * \param log2BlockSize log2 value of block size (square block) 2107 2034 * \param width width of the block 2108 2035 * \param height height of the block … … 2110 2037 * \returns ctxInc for current scan position 2111 2038 */ 2112 Int TComTrQuant::getSigCtxInc ( TCoeff* pcCoeff, 2039 Int TComTrQuant::getSigCtxInc ( 2040 Int patternSigCtx, 2041 UInt scanIdx, 2113 2042 Int posX, 2114 2043 Int posY, 2115 Int blockType, 2116 Int width 2117 ,Int height 2118 ,TextType textureType 2044 Int log2BlockSize, 2045 TextType textureType 2119 2046 ) 2120 2047 { 2121 if ( blockType == 2 ) 2122 { 2123 //LUMA map 2124 const Int ctxIndMap4x4Luma[15] = 2125 { 2126 0, 1, 4, 5, 2127 2, 3, 4, 5, 2128 6, 6, 8, 8, 2129 7, 7, 8 2130 }; 2131 //CHROMA map 2132 const Int ctxIndMap4x4Chroma[15] = 2133 { 2134 0, 1, 2, 4, 2135 1, 1, 2, 4, 2136 3, 3, 5, 5, 2137 4, 4, 5 2138 }; 2139 2140 if (textureType == TEXT_LUMA) 2141 { 2142 return ctxIndMap4x4Luma[ 4 * posY + posX ]; 2143 } 2144 else 2145 { 2146 return ctxIndMap4x4Chroma[ 4 * posY + posX ]; 2147 } 2148 } 2149 2150 if ( blockType == 3 ) 2151 { 2152 const Int map8x8[16] = 2153 { 2154 0, 1, 2, 3, 2155 4, 5, 6, 3, 2156 8, 6, 6, 7, 2157 9, 9, 7, 7 2158 }; 2159 2160 Int offset = (textureType == TEXT_LUMA) ? 9 : 6; 2161 2162 if ( posX + posY == 0 ) 2163 { 2164 return offset + 10; 2165 } 2166 return offset + map8x8[4 * (posY >> 1) + (posX >> 1)]; 2167 } 2168 2169 Int offset = (textureType == TEXT_LUMA) ? 20 : 17; 2048 const Int ctxIndMap[16] = 2049 { 2050 0, 1, 4, 5, 2051 2, 3, 4, 5, 2052 6, 6, 8, 8, 2053 7, 7, 8, 8 2054 }; 2055 2170 2056 if( posX + posY == 0 ) 2171 2057 { 2172 return offset; 2173 } 2174 #if SIGMAP_CONST_AT_HIGH_FREQUENCY 2175 Int thredHighFreq = 3*(std::max(width, height)>>4); 2176 if ((posX>>2) + (posY>>2) >= thredHighFreq) 2177 { 2178 return (textureType == TEXT_LUMA) ? 24 : 18; 2179 } 2180 #endif 2181 2182 const TCoeff *pData = pcCoeff + posX + posY * width; 2183 2184 2058 return 0; 2059 } 2060 2061 if ( log2BlockSize == 2 ) 2062 { 2063 return ctxIndMap[ 4 * posY + posX ]; 2064 } 2065 2066 Int offset = log2BlockSize == 3 ? (scanIdx==SCAN_DIAG ? 9 : 15) : (textureType == TEXT_LUMA ? 21 : 12); 2067 2068 Int posXinSubset = posX((posX>>2)<<2); 2069 Int posYinSubset = posY((posY>>2)<<2); 2185 2070 Int cnt = 0; 2186 if( posX < width  1 ) 2187 { 2188 cnt += pData[1] != 0; 2189 if( posY < height  1 ) 2190 { 2191 cnt += pData[width+1] != 0; 2192 } 2193 if( posX < width  2 ) 2194 { 2195 cnt += pData[2] != 0; 2196 } 2197 } 2198 if ( posY < height  1 ) 2199 { 2200 if( ( ( posX & 3 )  ( posY & 3 ) ) && ( ( (posX+1) & 3 )  ( (posY+2) & 3 ) ) ) 2201 { 2202 cnt += pData[width] != 0; 2203 } 2204 if ( posY < height  2 && cnt < 4 ) 2205 { 2206 cnt += pData[2*width] != 0; 2207 } 2208 } 2209 2210 cnt = ( cnt + 1 ) >> 1; 2211 return (( textureType == TEXT_LUMA && ((posX>>2) + (posY>>2)) > 0 ) ? 4 : 1) + offset + cnt; 2071 if(patternSigCtx==0) 2072 { 2073 cnt = posXinSubset+posYinSubset<=2 ? (posXinSubset+posYinSubset==0 ? 2 : 1) : 0; 2074 } 2075 else if(patternSigCtx==1) 2076 { 2077 cnt = posYinSubset<=1 ? (posYinSubset==0 ? 2 : 1) : 0; 2078 } 2079 else if(patternSigCtx==2) 2080 { 2081 cnt = posXinSubset<=1 ? (posXinSubset==0 ? 2 : 1) : 0; 2082 } 2083 else 2084 { 2085 cnt = 2; 2086 } 2087 2088 return (( textureType == TEXT_LUMA && ((posX>>2) + (posY>>2)) > 0 ) ? 3 : 0) + offset + cnt; 2212 2089 } 2213 2090 … … 2302 2179 2303 2180 if ( uiAbsLevel >= baseLevel ) 2304 { 2305 UInt uiSymbol = uiAbsLevel  baseLevel;2306 UInt uiMaxVlc = g_auiGoRiceRange[ ui16AbsGoRice ];2307 Bool bExpGolomb = ( uiSymbol > uiMaxVlc );2308 2309 if( bExpGolomb )2310 {2311 uiAbsLevel = uiSymbol  uiMaxVlc;2312 int iEGS = 1; for( UInt uiMax = 2; uiAbsLevel >= uiMax; uiMax <<= 1, iEGS += 2 );2313 iRate += iEGS << 15;2314 uiSymbol = min<UInt>( uiSymbol, ( uiMaxVlc + 1 ) );2315 }2316 2317 UShort ui16PrefLen = UShort( uiSymbol >> ui16AbsGoRice ) + 1;2318 UShort ui16NumBins = min<UInt>( ui16PrefLen, g_auiGoRicePrefixLen[ ui16AbsGoRice ] ) + ui16AbsGoRice;2319 2320 iRate += ui16NumBins<< 15;2321 2181 { 2182 UInt symbol = uiAbsLevel  baseLevel; 2183 UInt length; 2184 if (symbol < (COEF_REMAIN_BIN_REDUCTION << ui16AbsGoRice)) 2185 { 2186 length = symbol>>ui16AbsGoRice; 2187 iRate += (length+1+ui16AbsGoRice)<< 15; 2188 } 2189 else 2190 { 2191 length = ui16AbsGoRice; 2192 symbol = symbol  ( COEF_REMAIN_BIN_REDUCTION << ui16AbsGoRice); 2193 while (symbol >= (1<<length)) 2194 { 2195 symbol = (1<<(length++)); 2196 } 2197 iRate += (COEF_REMAIN_BIN_REDUCTION+length+1ui16AbsGoRice+length)<< 15; 2198 } 2322 2199 if (c1Idx < C1FLAG_NUMBER) 2323 2200 { … … 2367 2244 { 2368 2245 uiAbsLevel = uiSymbol  uiMaxVlc; 2369 int iEGS = 1; for( UInt uiMax = 2; uiAbsLevel >= uiMax; uiMax <<= 1, iEGS += 2 );2246 Int iEGS = 1; for( UInt uiMax = 2; uiAbsLevel >= uiMax; uiMax <<= 1, iEGS += 2 ); 2370 2247 iRate += iEGS << 15; 2371 2248 uiSymbol = min<UInt>( uiSymbol, ( uiMaxVlc + 1 ) ); … … 2423 2300 */ 2424 2301 __inline Double TComTrQuant::xGetRateLast ( const UInt uiPosX, 2425 const UInt uiPosY, 2426 const UInt uiBlkWdth ) const 2302 const UInt uiPosY ) const 2427 2303 { 2428 2304 UInt uiCtxX = g_uiGroupIdx[uiPosX]; … … 2480 2356 const UInt uiCGPosX, 2481 2357 const UInt uiCGPosY, 2482 const UInt scanIdx,2483 2358 Int width, Int height) 2484 2359 { … … 2488 2363 width >>= 2; 2489 2364 height >>= 2; 2490 if( width == 2 && height == 2 ) // 8x82491 {2492 if( scanIdx == SCAN_HOR )2493 {2494 width = 1;2495 height = 4;2496 }2497 else if( scanIdx == SCAN_VER )2498 {2499 width = 4;2500 height = 1;2501 }2502 }2503 2365 if( uiCGPosX < width  1 ) 2504 2366 { … … 2528 2390 xSetScalingListEnc(scalingList,list,size,qp); 2529 2391 xSetScalingListDec(scalingList,list,size,qp); 2530 setErrScaleCoeff(list,size,qp,SCALING_LIST_SQT); 2531 if(size == SCALING_LIST_32x32  size == SCALING_LIST_16x16) 2532 { 2533 setErrScaleCoeff(list,size1,qp,SCALING_LIST_HOR); 2534 setErrScaleCoeff(list,size1,qp,SCALING_LIST_VER); 2535 } 2392 setErrScaleCoeff(list,size,qp); 2536 2393 } 2537 2394 } … … 2562 2419 * \param uiQP Quantization parameter 2563 2420 */ 2564 Void TComTrQuant::setErrScaleCoeff(UInt list,UInt size, UInt qp , UInt dir)2421 Void TComTrQuant::setErrScaleCoeff(UInt list,UInt size, UInt qp) 2565 2422 { 2566 2423 2567 2424 UInt uiLog2TrSize = g_aucConvertToBit[ g_scalingListSizeX[size] ] + 2; 2568 #if FULL_NBIT 2569 UInt uiBitDepth = g_uiBitDepth; 2570 #else 2571 UInt uiBitDepth = g_uiBitDepth + g_uiBitIncrement; 2572 #endif 2573 2574 Int iTransformShift = MAX_TR_DYNAMIC_RANGE  uiBitDepth  uiLog2TrSize; // Represents scaling through forward transform 2425 Int bitDepth = (size < SCALING_LIST_32x32 && list != 0 && list != 3) ? g_bitDepthC : g_bitDepthY; 2426 Int iTransformShift = MAX_TR_DYNAMIC_RANGE  bitDepth  uiLog2TrSize; // Represents scaling through forward transform 2575 2427 2576 2428 UInt i,uiMaxNumCoeff = g_scalingListSize[size]; 2577 2429 Int *piQuantcoeff; 2578 double *pdErrScale;2579 piQuantcoeff = getQuantCoeff(list, qp,size ,dir);2580 pdErrScale = getErrScaleCoeff(list, size, qp ,dir);2581 2582 double dErrScale = (double)(1<<SCALE_BITS); // Compensate for scaling of bitcount in Lagrange cost function2430 Double *pdErrScale; 2431 piQuantcoeff = getQuantCoeff(list, qp,size); 2432 pdErrScale = getErrScaleCoeff(list, size, qp); 2433 2434 Double dErrScale = (Double)(1<<SCALE_BITS); // Compensate for scaling of bitcount in Lagrange cost function 2583 2435 dErrScale = dErrScale*pow(2.0,2.0*iTransformShift); // Compensate for scaling through forward transform 2584 2436 for(i=0;i<uiMaxNumCoeff;i++) 2585 2437 { 2586 pdErrScale[i] = dErrScale/(double)piQuantcoeff[i]/(double)piQuantcoeff[i]/(double)(1<<(2*g_uiBitIncrement));2438 pdErrScale[i] = dErrScale / piQuantcoeff[i] / piQuantcoeff[i] / (1<<DISTORTION_PRECISION_ADJUSTMENT(2*(bitDepth8))); 2587 2439 } 2588 2440 } … … 2601 2453 Int *quantcoeff; 2602 2454 Int *coeff = scalingList>getScalingListAddress(sizeId,listId); 2603 quantcoeff = getQuantCoeff(listId, qp, sizeId , SCALING_LIST_SQT);2455 quantcoeff = getQuantCoeff(listId, qp, sizeId); 2604 2456 2605 2457 processScalingListEnc(coeff,quantcoeff,g_quantScales[qp]<<4,height,width,ratio,min(MAX_MATRIX_SIZE_NUM,(Int)g_scalingListSizeX[sizeId]),scalingList>getScalingListDC(sizeId,listId)); 2606 2607 if(sizeId == SCALING_LIST_32x32  sizeId == SCALING_LIST_16x16) //for NSQT2608 {2609 quantcoeff = getQuantCoeff(listId, qp, sizeId1,SCALING_LIST_VER);2610 processScalingListEnc(coeff,quantcoeff,g_quantScales[qp]<<4,height,width>>2,ratio,min(MAX_MATRIX_SIZE_NUM,(Int)g_scalingListSizeX[sizeId]),scalingList>getScalingListDC(sizeId,listId));2611 2612 quantcoeff = getQuantCoeff(listId, qp, sizeId1,SCALING_LIST_HOR);2613 processScalingListEnc(coeff,quantcoeff,g_quantScales[qp]<<4,height>>2,width,ratio,min(MAX_MATRIX_SIZE_NUM,(Int)g_scalingListSizeX[sizeId]),scalingList>getScalingListDC(sizeId,listId));2614 }2615 2458 } 2616 2459 /** set quantized matrix coefficient for decode … … 2628 2471 Int *coeff = scalingList>getScalingListAddress(sizeId,listId); 2629 2472 2630 dequantcoeff = getDequantCoeff(listId, qp, sizeId ,SCALING_LIST_SQT);2473 dequantcoeff = getDequantCoeff(listId, qp, sizeId); 2631 2474 processScalingListDec(coeff,dequantcoeff,g_invQuantScales[qp],height,width,ratio,min(MAX_MATRIX_SIZE_NUM,(Int)g_scalingListSizeX[sizeId]),scalingList>getScalingListDC(sizeId,listId)); 2632 2633 if(sizeId == SCALING_LIST_32x32  sizeId == SCALING_LIST_16x16)2634 {2635 dequantcoeff = getDequantCoeff(listId, qp, sizeId1,SCALING_LIST_VER);2636 processScalingListDec(coeff,dequantcoeff,g_invQuantScales[qp],height,width>>2,ratio,min(MAX_MATRIX_SIZE_NUM,(Int)g_scalingListSizeX[sizeId]),scalingList>getScalingListDC(sizeId,listId));2637 2638 dequantcoeff = getDequantCoeff(listId, qp, sizeId1,SCALING_LIST_HOR);2639 2640 processScalingListDec(coeff,dequantcoeff,g_invQuantScales[qp],height>>2,width,ratio,min(MAX_MATRIX_SIZE_NUM,(Int)g_scalingListSizeX[sizeId]),scalingList>getScalingListDC(sizeId,listId));2641 }2642 2475 } 2643 2476 … … 2656 2489 { 2657 2490 xsetFlatScalingList(list,size,qp); 2658 setErrScaleCoeff(list,size,qp,SCALING_LIST_SQT); 2659 if(size == SCALING_LIST_32x32  size == SCALING_LIST_16x16) 2660 { 2661 setErrScaleCoeff(list,size1,qp,SCALING_LIST_HOR); 2662 setErrScaleCoeff(list,size1,qp,SCALING_LIST_VER); 2663 } 2491 setErrScaleCoeff(list,size,qp); 2664 2492 } 2665 2493 } … … 2675 2503 { 2676 2504 UInt i,num = g_scalingListSize[size]; 2677 UInt numDiv4 = num>>2;2678 2505 Int *quantcoeff; 2679 2506 Int *dequantcoeff; … … 2681 2508 Int invQuantScales = g_invQuantScales[qp]<<4; 2682 2509 2683 quantcoeff = getQuantCoeff(list, qp, size ,SCALING_LIST_SQT);2684 dequantcoeff = getDequantCoeff(list, qp, size ,SCALING_LIST_SQT);2510 quantcoeff = getQuantCoeff(list, qp, size); 2511 dequantcoeff = getDequantCoeff(list, qp, size); 2685 2512 2686 2513 for(i=0;i<num;i++) … … 2688 2515 *quantcoeff++ = quantScales; 2689 2516 *dequantcoeff++ = invQuantScales; 2690 }2691 2692 if(size == SCALING_LIST_32x32  size == SCALING_LIST_16x16)2693 {2694 quantcoeff = getQuantCoeff(list, qp, size1, SCALING_LIST_HOR);2695 dequantcoeff = getDequantCoeff(list, qp, size1, SCALING_LIST_HOR);2696 2697 for(i=0;i<numDiv4;i++)2698 {2699 *quantcoeff++ = quantScales;2700 *dequantcoeff++ = invQuantScales;2701 }2702 quantcoeff = getQuantCoeff(list, qp, size1 ,SCALING_LIST_VER);2703 dequantcoeff = getDequantCoeff(list, qp, size1 ,SCALING_LIST_VER);2704 2705 for(i=0;i<numDiv4;i++)2706 {2707 *quantcoeff++ = quantScales;2708 *dequantcoeff++ = invQuantScales;2709 }2710 2517 } 2711 2518 } … … 2749 2556 Void TComTrQuant::processScalingListDec( Int *coeff, Int *dequantcoeff, Int invQuantScales, UInt height, UInt width, UInt ratio, Int sizuNum, UInt dc) 2750 2557 { 2751 Int nsqth = (height < width) ? 4: 1; //height ratio for NSQT2752 Int nsqtw = (width < height) ? 4: 1; //width ratio for NSQT2753 2558 for(UInt j=0;j<height;j++) 2754 2559 { 2755 2560 for(UInt i=0;i<width;i++) 2756 2561 { 2757 dequantcoeff[j*width + i] = invQuantScales * coeff[sizuNum * (j * nsqth / ratio) + i * nsqtw /ratio];2562 dequantcoeff[j*width + i] = invQuantScales * coeff[sizuNum * (j / ratio) + i / ratio]; 2758 2563 } 2759 2564 } … … 2774 2579 for(UInt qp = 0; qp < SCALING_LIST_REM_NUM; qp++) 2775 2580 { 2776 m_quantCoef [sizeId][listId][qp][SCALING_LIST_SQT] = new Int [g_scalingListSize[sizeId]]; 2777 m_dequantCoef [sizeId][listId][qp][SCALING_LIST_SQT] = new Int [g_scalingListSize[sizeId]]; 2778 m_errScale [sizeId][listId][qp][SCALING_LIST_SQT] = new double [g_scalingListSize[sizeId]]; 2779 2780 if(sizeId == SCALING_LIST_8x8  (sizeId == SCALING_LIST_16x16 && listId < 2)) 2781 { 2782 for(UInt dir = SCALING_LIST_VER; dir < SCALING_LIST_DIR_NUM; dir++) 2783 { 2784 m_quantCoef [sizeId][listId][qp][dir] = new Int [g_scalingListSize[sizeId]]; 2785 m_dequantCoef [sizeId][listId][qp][dir] = new Int [g_scalingListSize[sizeId]]; 2786 m_errScale [sizeId][listId][qp][dir] = new double [g_scalingListSize[sizeId]]; 2787 } 2788 } 2789 } 2790 } 2791 } 2792 //copy for NSQT 2581 m_quantCoef [sizeId][listId][qp] = new Int [g_scalingListSize[sizeId]]; 2582 m_dequantCoef [sizeId][listId][qp] = new Int [g_scalingListSize[sizeId]]; 2583 m_errScale [sizeId][listId][qp] = new Double [g_scalingListSize[sizeId]]; 2584 } 2585 } 2586 } 2587 // alias list [1] as [3]. 2793 2588 for(UInt qp = 0; qp < SCALING_LIST_REM_NUM; qp++) 2794 2589 { 2795 for(UInt dir = SCALING_LIST_VER; dir < SCALING_LIST_DIR_NUM; dir++) 2796 { 2797 m_quantCoef [SCALING_LIST_16x16][3][qp][dir] = m_quantCoef [SCALING_LIST_16x16][1][qp][dir]; 2798 m_dequantCoef [SCALING_LIST_16x16][3][qp][dir] = m_dequantCoef [SCALING_LIST_16x16][1][qp][dir]; 2799 m_errScale [SCALING_LIST_16x16][3][qp][dir] = m_errScale [SCALING_LIST_16x16][1][qp][dir]; 2800 } 2801 m_quantCoef [SCALING_LIST_32x32][3][qp][SCALING_LIST_SQT] = m_quantCoef [SCALING_LIST_32x32][1][qp][SCALING_LIST_SQT]; 2802 m_dequantCoef [SCALING_LIST_32x32][3][qp][SCALING_LIST_SQT] = m_dequantCoef [SCALING_LIST_32x32][1][qp][SCALING_LIST_SQT]; 2803 m_errScale [SCALING_LIST_32x32][3][qp][SCALING_LIST_SQT] = m_errScale [SCALING_LIST_32x32][1][qp][SCALING_LIST_SQT]; 2590 m_quantCoef [SCALING_LIST_32x32][3][qp] = m_quantCoef [SCALING_LIST_32x32][1][qp]; 2591 m_dequantCoef [SCALING_LIST_32x32][3][qp] = m_dequantCoef [SCALING_LIST_32x32][1][qp]; 2592 m_errScale [SCALING_LIST_32x32][3][qp] = m_errScale [SCALING_LIST_32x32][1][qp]; 2804 2593 } 2805 2594 } … … 2814 2603 for(UInt qp = 0; qp < SCALING_LIST_REM_NUM; qp++) 2815 2604 { 2816 if(m_quantCoef [sizeId][listId][qp][SCALING_LIST_SQT]) delete [] m_quantCoef [sizeId][listId][qp][SCALING_LIST_SQT]; 2817 if(m_dequantCoef [sizeId][listId][qp][SCALING_LIST_SQT]) delete [] m_dequantCoef [sizeId][listId][qp][SCALING_LIST_SQT]; 2818 if(m_errScale [sizeId][listId][qp][SCALING_LIST_SQT]) delete [] m_errScale [sizeId][listId][qp][SCALING_LIST_SQT]; 2819 if(sizeId == SCALING_LIST_8x8  (sizeId == SCALING_LIST_16x16 && listId < 2)) 2820 { 2821 for(UInt dir = SCALING_LIST_VER; dir < SCALING_LIST_DIR_NUM; dir++) 2822 { 2823 if(m_quantCoef [sizeId][listId][qp][dir]) delete [] m_quantCoef [sizeId][listId][qp][dir]; 2824 if(m_dequantCoef [sizeId][listId][qp][dir]) delete [] m_dequantCoef [sizeId][listId][qp][dir]; 2825 if(m_errScale [sizeId][listId][qp][dir]) delete [] m_errScale [sizeId][listId][qp][dir]; 2826 } 2827 } 2605 if(m_quantCoef [sizeId][listId][qp]) delete [] m_quantCoef [sizeId][listId][qp]; 2606 if(m_dequantCoef [sizeId][listId][qp]) delete [] m_dequantCoef [sizeId][listId][qp]; 2607 if(m_errScale [sizeId][listId][qp]) delete [] m_errScale [sizeId][listId][qp]; 2828 2608 } 2829 2609 }
Note: See TracChangeset for help on using the changeset viewer.