This encoder issue does not affect common test condition results.

In HM-5.0 1-pass ALF encoding, the filtering distortion estimation does not consider the 2 skipped lines per LCU if STAR_5x5 filter shape is selected. This problem may cause the filter shape decision biased when 1/2-pass ALF encoding is enabled.

The fix is included by #if FIX_ONE_PASS in the following codes. Only one function needs to be modified: TEncAdaptiveLoopFilter::decideFilterShapeLuma

======
Void TEncAdaptiveLoopFilter::decideFilterShapeLuma(Pel* ImgOrg, Pel* ImgDec, Int Stride, ALFParam* pcAlfSaved, UInt64& ruiRate, UInt64& ruiDist, Double& rdCost)
{

static Double ySym, *ESym;
Int lambda_val = ((Int) m_dLambdaLuma) * (1<<(2*g_uiBitIncrement));
Int filtNo, filters_per_fr;
Int64 iEstimatedDist;
UInt64 uiRate;
Double dEstimatedCost, dEstimatedMinCost = MAX_DOUBLE;;

UInt uiBitShift = (g_uiBitIncrement<<1);

#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB

Int64 iEstimateDistBeforeFilter;
Int* coeffNoFilter[NUM_ALF_FILTER_SHAPE][NO_VAR_BINS];
for(Int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE; filter_shape++)
{

for(Int i=0; i< NO_VAR_BINS; i++)
{

coeffNoFilter[filter_shape][i]= new Int[ALF_MAX_NUM_COEF];
::memset(coeffNoFilter[filter_shape][i], 0, sizeof(Int)*ALF_MAX_NUM_COEF);

#if ALF_DC_OFFSET_REMOVAL

coeffNoFilter[filter_shape][i][ m_sqrFiltLengthTab[filter_shape]-1 ] = (1 << ((Int)ALF_NUM_BIT_SHIFT));

#else

coeffNoFilter[filter_shape][i][ m_sqrFiltLengthTab[filter_shape]-2 ] = (1 << ((Int)ALF_NUM_BIT_SHIFT));

#endif

}

}

#endif
#endif

m_pcTempAlfParam->alf_flag = 1;

#if !F747_APS

m_pcTempAlfParam->cu_control_flag = 0;

#endif

m_pcTempAlfParam->chroma_idc = 0;
m_pcTempAlfParam->alf_pcr_region_flag = m_uiVarGenMethod;

#if !G212_CROSS9x9_VB

calculate correlation matrix (11x5)
if(!m_bUseNonCrossALF)
{

xstoreInBlockMatrix(0, 0, m_img_height, m_img_width, true, true, ImgOrg, ImgDec, 2, Stride);

}
else
{

xstoreInBlockMatrixforSlices(ImgOrg, ImgDec, 2, Stride);

}

#endif

for (int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE ;filter_shape ++)
{

m_pcTempAlfParam->filter_shape = filtNo = filter_shape;
m_pcTempAlfParam->num_coeff = m_sqrFiltLengthTab[filtNo] ;

ESym = m_EGlobalSym [filtNo];
ySym = m_yGlobalSym [filtNo];

#if G212_CROSS9x9_VB

if(!m_bUseNonCrossALF)
{

xstoreInBlockMatrix(0, 0, m_img_height, m_img_width, true, true, ImgOrg, ImgDec, filter_shape, Stride);

}
else
{

xstoreInBlockMatrixforSlices(ImgOrg, ImgDec, filter_shape, Stride);

}

#else

xretriveBlockMatrix(m_pcTempAlfParam->num_coeff, m_iFilterTabIn11x5Sym[filtNo], m_EGlobalSym[2], ESym, m_yGlobalSym[2], ySym);

#endif

xfindBestFilterVarPred(ySym, ESym, m_pixAcc, m_filterCoeffSym, m_filterCoeffSymQuant, filtNo, &filters_per_fr, m_varIndTab, NULL, m_varImg, m_maskImg, NULL, lambda_val);

estimate R-D cost
uiRate = xcodeFiltCoeff(m_filterCoeffSymQuant, filtNo, m_varIndTab, filters_per_fr, m_pcTempAlfParam);
iEstimatedDist = xEstimateFiltDist(filters_per_fr, m_varIndTab, ESym, ySym, m_filterCoeffSym, m_pcTempAlfParam->num_coeff);

#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB

iEstimateDistBeforeFilter = xEstimateFiltDist(filters_per_fr, m_varIndTab, ESym, ySym, coeffNoFilter[filter_shape], m_pcTempAlfParam->num_coeff);
iEstimatedDist -= iEstimateDistBeforeFilter;

#endif
#endif

dEstimatedCost = (Double)(uiRate) * m_dLambdaLuma + (Double)(iEstimatedDist);

if(dEstimatedCost < dEstimatedMinCost)
{

dEstimatedMinCost = dEstimatedCost;
copyALFParam(pcAlfSaved, m_pcTempAlfParam);

#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB

iEstimatedDist += iEstimateDistBeforeFilter;

#endif
#endif

for(Int i=0; i< filters_per_fr; i++ )
{

iEstimatedDist += (((Int64)m_pixAcc_merged[i]) >> uiBitShift);

}
ruiDist = (iEstimatedDist > 0)?((UInt64)iEstimatedDist):(0);
rdCost = dEstimatedMinCost + (Double)(ruiDist);
ruiRate = uiRate;

}

}

if (!m_iUsePreviousFilter)
{

decodeFilterSet(pcAlfSaved, m_varIndTab, m_filterCoeffSym);
saveFilterCoeffToBuffer(m_filterCoeffSym, pcAlfSaved->filters_per_group, m_varIndTab, pcAlfSaved->alf_pcr_region_flag, pcAlfSaved->filter_shape);

}

if( m_iUsePreviousFilter )
{

UInt64 uiOffRegionDistortion = 0;
Int iPelDiff;
Pel* pOrgTemp = (Pel*)ImgOrg;
Pel* pDecTemp = (Pel*)ImgDec;
for(Int y=0; y< m_img_height; y++)
{

for(Int x=0; x< m_img_width; x++)
{

if(m_maskImg[y][x] == 0)
{

iPelDiff = pOrgTemp[x] - pDecTemp[x];
uiOffRegionDistortion += (UInt64)( (iPelDiff*iPelDiff) >> uiBitShift );

}

}
pOrgTemp += Stride;
pDecTemp += Stride;

ruiDist += uiOffRegionDistortion;
rdCost += (Double)uiOffRegionDistortion;

}

}

#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB

if(pcAlfSaved->filter_shape == ALF_STAR5x5)
{

Int iPelDiff;
UInt64 uiSkipPelsDistortion = 0;
Pel *pOrgTemp, *pDecTemp;
for(Int y= m_lineIdxPadTop-1; y< m_img_height - m_lcuHeight ; y += m_lcuHeight)
{

pOrgTemp = ImgOrg + y*Stride;
pDecTemp = ImgDec + y*Stride;
for(Int x=0; x< m_img_width; x++)
{

if(m_maskImg[y][x] == 1)
{

iPelDiff = pOrgTemp[x] - pDecTemp[x];
uiSkipPelsDistortion += (UInt64)( (iPelDiff*iPelDiff) >> uiBitShift );

}

}

pOrgTemp += Stride;
pDecTemp += Stride;
for(Int x=0; x< m_img_width; x++)
{

if(m_maskImg[y+1][x] == 1)
{

iPelDiff = pOrgTemp[x] - pDecTemp[x];
uiSkipPelsDistortion += (UInt64)( (iPelDiff*iPelDiff) >> uiBitShift );

}

}

}
ruiDist += uiSkipPelsDistortion;
rdCost += (Double)uiSkipPelsDistortion;

}

for(Int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE; filter_shape++)
{

for(Int i=0; i< NO_VAR_BINS; i++)
{

delete[] coeffNoFilter[filter_shape][i];

}

}

#endif
#endif

}