This encoder issue does not affect common test condition results.

In HM-5.0 1-pass ALF encoding, the filtering distortion estimation does not consider the 2 skipped lines per LCU if STAR_5x5 filter shape is selected. This problem may cause the filter shape decision biased when 1/2-pass ALF encoding is enabled.

The fix is included by #if FIX_ONE_PASS in the following codes. Only one function needs to be modified: TEncAdaptiveLoopFilter::decideFilterShapeLuma

Void  TEncAdaptiveLoopFilter::decideFilterShapeLuma(Pel* ImgOrg, Pel* ImgDec, Int Stride, ALFParam* pcAlfSaved, UInt64& ruiRate, UInt64& ruiDist, Double& rdCost)
{
  static Double **ySym, ***ESym;
  Int    lambda_val = ((Int) m_dLambdaLuma) * (1<<(2*g_uiBitIncrement));
  Int    filtNo, filters_per_fr;
  Int64  iEstimatedDist;
  UInt64 uiRate;
  Double dEstimatedCost, dEstimatedMinCost = MAX_DOUBLE;;

  UInt   uiBitShift = (g_uiBitIncrement<<1);

#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB
  Int64  iEstimateDistBeforeFilter;
  Int*   coeffNoFilter[NUM_ALF_FILTER_SHAPE][NO_VAR_BINS];
  for(Int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE; filter_shape++)
  {
    for(Int i=0; i< NO_VAR_BINS; i++)
    {
      coeffNoFilter[filter_shape][i]= new Int[ALF_MAX_NUM_COEF];
      ::memset(coeffNoFilter[filter_shape][i], 0, sizeof(Int)*ALF_MAX_NUM_COEF);
#if ALF_DC_OFFSET_REMOVAL
      coeffNoFilter[filter_shape][i][ m_sqrFiltLengthTab[filter_shape]-1 ] = (1 << ((Int)ALF_NUM_BIT_SHIFT));
#else
      coeffNoFilter[filter_shape][i][ m_sqrFiltLengthTab[filter_shape]-2 ] = (1 << ((Int)ALF_NUM_BIT_SHIFT));
#endif
    }
  }
#endif
#endif

  m_pcTempAlfParam->alf_flag = 1;
#if !F747_APS
  m_pcTempAlfParam->cu_control_flag = 0;
#endif
  m_pcTempAlfParam->chroma_idc = 0;
  m_pcTempAlfParam->alf_pcr_region_flag = m_uiVarGenMethod;

#if !G212_CROSS9x9_VB
  //calculate correlation matrix (11x5)
  if(!m_bUseNonCrossALF)
  {
    xstoreInBlockMatrix(0, 0, m_img_height, m_img_width, true, true, ImgOrg, ImgDec, 2, Stride);
  }
  else
  {
    xstoreInBlockMatrixforSlices(ImgOrg, ImgDec, 2, Stride);
  }
#endif

  for (int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE ;filter_shape ++)
  {
    m_pcTempAlfParam->filter_shape = filtNo = filter_shape;
    m_pcTempAlfParam->num_coeff = m_sqrFiltLengthTab[filtNo] ;  

    ESym     = m_EGlobalSym     [filtNo];
    ySym     = m_yGlobalSym     [filtNo];

#if G212_CROSS9x9_VB
    if(!m_bUseNonCrossALF)
    {
      xstoreInBlockMatrix(0, 0, m_img_height, m_img_width, true, true, ImgOrg, ImgDec, filter_shape, Stride);
    }
    else
    {
      xstoreInBlockMatrixforSlices(ImgOrg, ImgDec, filter_shape, Stride);
    }

#else
    xretriveBlockMatrix(m_pcTempAlfParam->num_coeff, m_iFilterTabIn11x5Sym[filtNo], m_EGlobalSym[2], ESym, m_yGlobalSym[2], ySym);
#endif
    xfindBestFilterVarPred(ySym, ESym, m_pixAcc, m_filterCoeffSym, m_filterCoeffSymQuant, filtNo, &filters_per_fr, m_varIndTab, NULL, m_varImg, m_maskImg, NULL, lambda_val);

    //estimate R-D cost
    uiRate         = xcodeFiltCoeff(m_filterCoeffSymQuant, filtNo, m_varIndTab, filters_per_fr, m_pcTempAlfParam);
    iEstimatedDist = xEstimateFiltDist(filters_per_fr, m_varIndTab, ESym, ySym, m_filterCoeffSym, m_pcTempAlfParam->num_coeff);
#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB
    iEstimateDistBeforeFilter = xEstimateFiltDist(filters_per_fr, m_varIndTab, ESym, ySym, coeffNoFilter[filter_shape], m_pcTempAlfParam->num_coeff);
    iEstimatedDist -= iEstimateDistBeforeFilter;
#endif
#endif
    dEstimatedCost = (Double)(uiRate) * m_dLambdaLuma + (Double)(iEstimatedDist);

    if(dEstimatedCost < dEstimatedMinCost)
    {
      dEstimatedMinCost   = dEstimatedCost;
      copyALFParam(pcAlfSaved, m_pcTempAlfParam); 
#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB
      iEstimatedDist += iEstimateDistBeforeFilter;
#endif
#endif
      for(Int i=0; i< filters_per_fr; i++ )
      {
        iEstimatedDist += (((Int64)m_pixAcc_merged[i]) >> uiBitShift);
      }
      ruiDist = (iEstimatedDist > 0)?((UInt64)iEstimatedDist):(0);
      rdCost  = dEstimatedMinCost + (Double)(ruiDist);
      ruiRate = uiRate;
    }
  }

  if (!m_iUsePreviousFilter)
  {
    decodeFilterSet(pcAlfSaved, m_varIndTab, m_filterCoeffSym);
    saveFilterCoeffToBuffer(m_filterCoeffSym, pcAlfSaved->filters_per_group, m_varIndTab, pcAlfSaved->alf_pcr_region_flag, pcAlfSaved->filter_shape);
  }

  if( m_iUsePreviousFilter )
  {
    UInt64 uiOffRegionDistortion = 0;
    Int    iPelDiff;
    Pel*   pOrgTemp = (Pel*)ImgOrg;
    Pel*   pDecTemp = (Pel*)ImgDec;
    for(Int y=0; y< m_img_height; y++)
    {
      for(Int x=0; x< m_img_width; x++)
      {
        if(m_maskImg[y][x] == 0)
        {
          iPelDiff = pOrgTemp[x] - pDecTemp[x];
          uiOffRegionDistortion += (UInt64)(  (iPelDiff*iPelDiff) >> uiBitShift );
        }
      }
      pOrgTemp += Stride;
      pDecTemp += Stride;

      ruiDist += uiOffRegionDistortion;
      rdCost  += (Double)uiOffRegionDistortion;
    }
  }

#if FIX_ONE_PASS
#if G1023_FIX_NPASS_ALF && G212_CROSS9x9_VB
  if(pcAlfSaved->filter_shape == ALF_STAR5x5)
  {
    Int    iPelDiff;
    UInt64  uiSkipPelsDistortion = 0;
    Pel   *pOrgTemp, *pDecTemp;
    for(Int y= m_lineIdxPadTop-1; y< m_img_height - m_lcuHeight ; y += m_lcuHeight)
    {
      pOrgTemp = ImgOrg + y*Stride;
      pDecTemp = ImgDec + y*Stride;
      for(Int x=0; x< m_img_width; x++)
      {
        if(m_maskImg[y][x] == 1)
        {
          iPelDiff = pOrgTemp[x] - pDecTemp[x];
          uiSkipPelsDistortion += (UInt64)(  (iPelDiff*iPelDiff) >> uiBitShift );
        }
      }

      pOrgTemp += Stride;
      pDecTemp += Stride;
      for(Int x=0; x< m_img_width; x++)
      {
        if(m_maskImg[y+1][x] == 1)
        {
          iPelDiff = pOrgTemp[x] - pDecTemp[x];
          uiSkipPelsDistortion += (UInt64)(  (iPelDiff*iPelDiff) >> uiBitShift );
        }
      }
    }
    ruiDist += uiSkipPelsDistortion;
    rdCost  += (Double)uiSkipPelsDistortion;
  }

  for(Int filter_shape = 0; filter_shape < NUM_ALF_FILTER_SHAPE; filter_shape++)
  {
    for(Int i=0; i< NO_VAR_BINS; i++)
    {
      delete[] coeffNoFilter[filter_shape][i];
    }
  }
#endif
#endif

}