Ticket #539: deblocker_bug.patch
File deblocker_bug.patch, 4.7 KB (added by bheng, 11 years ago) |
---|
-
Lib/TLibCommon/TComLoopFilter.cpp
219 219 Int iDir = Edge; 220 220 for( UInt uiPartIdx = uiAbsZorderIdx; uiPartIdx < uiAbsZorderIdx + uiCurNumParts; uiPartIdx++ ) 221 221 { 222 UInt uiBSCheck = ((iDir == EDGE_VER && uiPartIdx%2 == 0) || (iDir == EDGE_HOR && (uiPartIdx-((uiPartIdx>>2)<<2))/2 == 0)); 222 UInt uiBSCheck; 223 if( (g_uiMaxCUWidth >> g_uiMaxCUDepth) == 4 ) 224 { 225 uiBSCheck = (iDir == EDGE_VER && uiPartIdx%2 == 0) || (iDir == EDGE_HOR && (uiPartIdx-((uiPartIdx>>2)<<2))/2 == 0); 226 } 227 else { 228 uiBSCheck = 1; 229 } 230 223 231 if ( m_aapbEdgeFilter[iDir][0][uiPartIdx] && uiBSCheck ) 224 232 { 225 233 xGetBoundaryStrengthSingle ( pcCU, uiAbsZorderIdx, iDir, uiPartIdx ); … … 234 242 for ( UInt iEdge = 0; iEdge < uiSizeInPU ; iEdge+=PartIdxIncr) 235 243 { 236 244 xEdgeFilterLuma ( pcCU, uiAbsZorderIdx, uiDepth, iDir, iEdge ); 237 if ( ( iEdge % ( (DEBLOCK_SMALLEST_BLOCK<<1)/uiPelsInPart ) ) == 0 )245 if ( (uiPelsInPart>DEBLOCK_SMALLEST_BLOCK) || (iEdge % ( (DEBLOCK_SMALLEST_BLOCK<<1)/uiPelsInPart ) ) == 0 ) 238 246 { 239 247 xEdgeFilterChroma ( pcCU, uiAbsZorderIdx, uiDepth, iDir, iEdge ); 240 248 } … … 288 296 Int trHeight = pcCU->getHeight( uiAbsZorderIdx ) >> pcCU->getTransformIdx( uiAbsZorderIdx ); 289 297 pcCU->getNSQTSize( uiDepth - pcCU->getDepth( uiAbsZorderIdx ), uiAbsZorderIdx, trWidth, trHeight ); 290 298 291 UInt uiWidthInBaseUnits = trWidth >> 2;292 UInt uiHeightInBaseUnits = trHeight >> 2;299 UInt uiWidthInBaseUnits = trWidth / (g_uiMaxCUWidth >> g_uiMaxCUDepth); 300 UInt uiHeightInBaseUnits = trHeight / (g_uiMaxCUWidth >> g_uiMaxCUDepth); 293 301 294 302 xSetEdgefilterMultiple( pcCU, absTUPartIdx, uiDepth, EDGE_VER, 0, m_stLFCUParam.bInternalEdge, uiWidthInBaseUnits, uiHeightInBaseUnits ); 295 303 xSetEdgefilterMultiple( pcCU, absTUPartIdx, uiDepth, EDGE_HOR, 0, m_stLFCUParam.bInternalEdge, uiWidthInBaseUnits, uiHeightInBaseUnits ); … … 612 620 Int iBeta = betatable_8x8[iIndexB]*iBitdepthScale; 613 621 Int iSideThreshold = (iBeta+(iBeta>>1))>>3; 614 622 Int iThrCut = iTc*10; 615 616 Int dp0 = xCalcDP( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+0), iOffset); 617 Int dq0 = xCalcDQ( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+0), iOffset); 618 Int dp3 = xCalcDP( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+3), iOffset); 619 Int dq3 = xCalcDQ( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+3), iOffset); 623 624 UInt uiBlocksInPart = uiPelsInPart / 4 ? uiPelsInPart / 4 : 1; 625 for (UInt iBlkIdx = 0; iBlkIdx<uiBlocksInPart; iBlkIdx ++) 626 { 627 Int dp0 = xCalcDP( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+iBlkIdx*4+0), iOffset); 628 Int dq0 = xCalcDQ( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+iBlkIdx*4+0), iOffset); 629 Int dp3 = xCalcDP( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+iBlkIdx*4+3), iOffset); 630 Int dq3 = xCalcDQ( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+iBlkIdx*4+3), iOffset); 620 631 Int d0 = dp0 + dq0; 621 632 Int d3 = dp3 + dq3; 622 633 … … 640 651 Bool bFilterP = (dp < iSideThreshold); 641 652 Bool bFilterQ = (dq < iSideThreshold); 642 653 643 Bool sw = xUseStrongFiltering( iOffset, 2*d0, iBeta, iTc, piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+ 0))644 && xUseStrongFiltering( iOffset, 2*d3, iBeta, iTc, piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+ 3));654 Bool sw = xUseStrongFiltering( iOffset, 2*d0, iBeta, iTc, piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+iBlkIdx*4+0)) 655 && xUseStrongFiltering( iOffset, 2*d3, iBeta, iTc, piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+iBlkIdx*4+3)); 645 656 646 657 for ( Int i = 0; i < DEBLOCK_SMALLEST_BLOCK/2; i++) 647 658 { 648 xPelFilterLuma( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+i ), iOffset, d, iBeta, iTc, sw, bPartPNoFilter, bPartQNoFilter, iThrCut, bFilterP, bFilterQ);659 xPelFilterLuma( piTmpSrc+iSrcStep*(iIdx*uiPelsInPart+iBlkIdx*4+i), iOffset, d, iBeta, iTc, sw, bPartPNoFilter, bPartQNoFilter, iThrCut, bFilterP, bFilterQ); 649 660 650 661 } 651 662 } 663 } 652 664 } 653 665 } 654 666 } … … 682 694 UInt uiEdgeNumInLCUVert = g_auiZscanToRaster[uiAbsZorderIdx]%uiLCUWidthInBaseUnits + iEdge; 683 695 UInt uiEdgeNumInLCUHor = g_auiZscanToRaster[uiAbsZorderIdx]/uiLCUWidthInBaseUnits + iEdge; 684 696 685 if ( ( (uiEdgeNumInLCUVert%(DEBLOCK_SMALLEST_BLOCK/uiPelsInPartChroma))&&(iDir==0) ) || ( (uiEdgeNumInLCUHor%(DEBLOCK_SMALLEST_BLOCK/uiPelsInPartChroma))&& iDir ))697 if ( (uiPelsInPartChroma < DEBLOCK_SMALLEST_BLOCK) && (( (uiEdgeNumInLCUVert%(DEBLOCK_SMALLEST_BLOCK/uiPelsInPartChroma))&&(iDir==0) ) || ( (uiEdgeNumInLCUHor%(DEBLOCK_SMALLEST_BLOCK/uiPelsInPartChroma))&& iDir ) )) 686 698 { 687 699 return; 688 700 }