id summary reporter owner description type status priority milestone component version resolution keywords cc 285 1-pass ALF encoder issue due to correlation calculations chiayang_tsai "This encoder issue does not affect common test condition results. In HM-5.0 1-pass ALF encoding, the filtering distortion estimation does not consider the 2 skipped lines per LCU if STAR_5x5 filter shape is selected. This problem may cause the filter shape decision biased when 1/2-pass ALF encoding is enabled. The fix is included by #if FIX_ONE_PASS in the following codes. Only one function needs to be modified: TEncAdaptiveLoopFilter::decideFilterShapeLuma {{{ Void TEncAdaptiveLoopFilter::decideFilterShapeLuma(Pel* ImgOrg, Pel* ImgDec, Int Stride, ALFParam* pcAlfSaved, UInt64& ruiRate, UInt64& ruiDist, Double& rdCost) { static Double **ySym, ***ESym; Int lambda_val = ((Int) m_dLambdaLuma) * (1<<(2*g_uiBitIncrement)); Int filtNo, filters_per_fr; Int64 iEstimatedDist; UInt64 uiRate; Double dEstimatedCost, dEstimatedMinCost = MAX_DOUBLE;; UInt uiBitShift = (g_uiBitIncrement<<1); #if FIX_ONE_PASS #if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB Int64 iEstimateDistBeforeFilter; Int* coeffNoFilter[NUM_ALF_FILTER_SHAPE][NO_VAR_BINS]; for(Int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE; filter_shape++) { for(Int i=0; i< NO_VAR_BINS; i++) { coeffNoFilter[filter_shape][i]= new Int[ALF_MAX_NUM_COEF]; ::memset(coeffNoFilter[filter_shape][i], 0, sizeof(Int)*ALF_MAX_NUM_COEF); #if ALF_DC_OFFSET_REMOVAL coeffNoFilter[filter_shape][i][ m_sqrFiltLengthTab[filter_shape]-1 ] = (1 << ((Int)ALF_NUM_BIT_SHIFT)); #else coeffNoFilter[filter_shape][i][ m_sqrFiltLengthTab[filter_shape]-2 ] = (1 << ((Int)ALF_NUM_BIT_SHIFT)); #endif } } #endif #endif m_pcTempAlfParam->alf_flag = 1; #if !F747_APS m_pcTempAlfParam->cu_control_flag = 0; #endif m_pcTempAlfParam->chroma_idc = 0; m_pcTempAlfParam->alf_pcr_region_flag = m_uiVarGenMethod; #if !G212_CROSS9x9_VB //calculate correlation matrix (11x5) if(!m_bUseNonCrossALF) { xstoreInBlockMatrix(0, 0, m_img_height, m_img_width, true, true, ImgOrg, ImgDec, 2, Stride); } else { xstoreInBlockMatrixforSlices(ImgOrg, ImgDec, 2, Stride); } #endif for (int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE ;filter_shape ++) { m_pcTempAlfParam->filter_shape = filtNo = filter_shape; m_pcTempAlfParam->num_coeff = m_sqrFiltLengthTab[filtNo] ; ESym = m_EGlobalSym [filtNo]; ySym = m_yGlobalSym [filtNo]; #if G212_CROSS9x9_VB if(!m_bUseNonCrossALF) { xstoreInBlockMatrix(0, 0, m_img_height, m_img_width, true, true, ImgOrg, ImgDec, filter_shape, Stride); } else { xstoreInBlockMatrixforSlices(ImgOrg, ImgDec, filter_shape, Stride); } #else xretriveBlockMatrix(m_pcTempAlfParam->num_coeff, m_iFilterTabIn11x5Sym[filtNo], m_EGlobalSym[2], ESym, m_yGlobalSym[2], ySym); #endif xfindBestFilterVarPred(ySym, ESym, m_pixAcc, m_filterCoeffSym, m_filterCoeffSymQuant, filtNo, &filters_per_fr, m_varIndTab, NULL, m_varImg, m_maskImg, NULL, lambda_val); //estimate R-D cost uiRate = xcodeFiltCoeff(m_filterCoeffSymQuant, filtNo, m_varIndTab, filters_per_fr, m_pcTempAlfParam); iEstimatedDist = xEstimateFiltDist(filters_per_fr, m_varIndTab, ESym, ySym, m_filterCoeffSym, m_pcTempAlfParam->num_coeff); #if FIX_ONE_PASS #if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB iEstimateDistBeforeFilter = xEstimateFiltDist(filters_per_fr, m_varIndTab, ESym, ySym, coeffNoFilter[filter_shape], m_pcTempAlfParam->num_coeff); iEstimatedDist -= iEstimateDistBeforeFilter; #endif #endif dEstimatedCost = (Double)(uiRate) * m_dLambdaLuma + (Double)(iEstimatedDist); if(dEstimatedCost < dEstimatedMinCost) { dEstimatedMinCost = dEstimatedCost; copyALFParam(pcAlfSaved, m_pcTempAlfParam); #if FIX_ONE_PASS #if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB iEstimatedDist += iEstimateDistBeforeFilter; #endif #endif for(Int i=0; i< filters_per_fr; i++ ) { iEstimatedDist += (((Int64)m_pixAcc_merged[i]) >> uiBitShift); } ruiDist = (iEstimatedDist > 0)?((UInt64)iEstimatedDist):(0); rdCost = dEstimatedMinCost + (Double)(ruiDist); ruiRate = uiRate; } } if (!m_iUsePreviousFilter) { decodeFilterSet(pcAlfSaved, m_varIndTab, m_filterCoeffSym); saveFilterCoeffToBuffer(m_filterCoeffSym, pcAlfSaved->filters_per_group, m_varIndTab, pcAlfSaved->alf_pcr_region_flag, pcAlfSaved->filter_shape); } if( m_iUsePreviousFilter ) { UInt64 uiOffRegionDistortion = 0; Int iPelDiff; Pel* pOrgTemp = (Pel*)ImgOrg; Pel* pDecTemp = (Pel*)ImgDec; for(Int y=0; y< m_img_height; y++) { for(Int x=0; x< m_img_width; x++) { if(m_maskImg[y][x] == 0) { iPelDiff = pOrgTemp[x] - pDecTemp[x]; uiOffRegionDistortion += (UInt64)( (iPelDiff*iPelDiff) >> uiBitShift ); } } pOrgTemp += Stride; pDecTemp += Stride; ruiDist += uiOffRegionDistortion; rdCost += (Double)uiOffRegionDistortion; } } #if FIX_ONE_PASS #if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB if(pcAlfSaved->filter_shape == ALF_STAR5x5) { Int iPelDiff; UInt64 uiSkipPelsDistortion = 0; Pel *pOrgTemp, *pDecTemp; for(Int y= m_lineIdxPadTop-1; y< m_img_height - m_lcuHeight ; y += m_lcuHeight) { pOrgTemp = ImgOrg + y*Stride; pDecTemp = ImgDec + y*Stride; for(Int x=0; x< m_img_width; x++) { if(m_maskImg[y][x] == 1) { iPelDiff = pOrgTemp[x] - pDecTemp[x]; uiSkipPelsDistortion += (UInt64)( (iPelDiff*iPelDiff) >> uiBitShift ); } } pOrgTemp += Stride; pDecTemp += Stride; for(Int x=0; x< m_img_width; x++) { if(m_maskImg[y+1][x] == 1) { iPelDiff = pOrgTemp[x] - pDecTemp[x]; uiSkipPelsDistortion += (UInt64)( (iPelDiff*iPelDiff) >> uiBitShift ); } } } ruiDist += uiSkipPelsDistortion; rdCost += (Double)uiSkipPelsDistortion; } for(Int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE; filter_shape++) { for(Int i=0; i< NO_VAR_BINS; i++) { delete[] coeffNoFilter[filter_shape][i]; } } #endif #endif } }}} " defect closed minor HM-5.1 HM HM-5.0 fixed fbossen ksuehring davidf jct-vc@…