source: 3DVCSoftware/trunk/source/App/utils/BitrateTargeting/targetBitrates.sh @ 1417

Last change on this file since 1417 was 872, checked in by tech, 11 years ago

Merged HTM-10.0-dev0@871. (MV-HEVC 7 HLS)
File size: 17.4 KB
Line 
1#! /bin/sh
2
3# The copyright in this software is being made available under the BSD
4# License, included below. This software may be subject to other third party
5# and contributor rights, including patent rights, and no such rights are
6# granted under this license. 
7#
8# Copyright (c) 2010-2014, ITU/ISO/IEC
9# All rights reserved.
10#
11# Redistribution and use in source and binary forms, with or without
12# modification, are permitted provided that the following conditions are met:
13#
14#  * Redistributions of source code must retain the above copyright notice,
15#    this list of conditions and the following disclaimer.
16#  * Redistributions in binary form must reproduce the above copyright notice,
17#    this list of conditions and the following disclaimer in the documentation
18#    and/or other materials provided with the distribution.
19#  * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
20#    be used to endorse or promote products derived from this software without
21#    specific prior written permission.
22#
23# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
27# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33# THE POSSIBILITY OF SUCH DAMAGE.
34
35
36SUB_TOOLS_DIRECTORY=$(echo "$0" | sed -e 's/[^\/]*$//')
37. ${SUB_TOOLS_DIRECTORY}encode.shl
38
39LAMBDA_MODIFIER_PREFIX="LM"
40
41RESUME_MODE_OPTION="-rm"
42TARGET_BITRATES_OPTION="-tb"
43INITIAL_LAMBDA_MODIFIERS_OPTION="-il"
44ENCODE_COMMAND_ARGS_OPTION="-ca"
45
46function outputUsageAndExit {
47  local TARGET_BITRATES_USAGE_STRING=targetBitrates
48  local INITIAL_LAMBDA_MODIFIERS_USAGE_STRING=initialLambdaModifiers
49  local ENCODE_COMMAND_ARGS_USAGE_STRING=encodeCommandArgs
50 
51  echo "Usage: $0 [$RESUME_MODE_OPTION] $CONFIGURATION_IDENTIFIER_OPTION $CONFIGURATION_IDENTIFIER_USAGE_STRING $Q_OPTION $Q_USAGE_STRING $TARGET_BITRATES_OPTION $TARGET_BITRATES_USAGE_STRING [$INITIAL_LAMBDA_MODIFIERS_OPTION $INITIAL_LAMBDA_MODIFIERS_USAGE_STRING] [$ENCODE_COMMAND_ARGS_OPTION $ENCODE_COMMAND_ARGS_USAGE_STRING] [$EXTRA_ARGUMENTS_OPTION $EXTRA_ARGUMENTS_USAGE_STRING] $OUTPUT_DIRECTORY_OPTION $OUTPUT_DIRECTORY_USAGE_STRING $INPUT_NAME_USAGE_STRING" >&2
52  echo "${USAGE_INDENT}$RESUME_MODE_OPTION engages resume mode which allows the user to resume an execution that was interrupted before completion."
53  outputConfigurationIdentifierUsage
54  outputQUsage
55  echo "${USAGE_INDENT}$TARGET_BITRATES_USAGE_STRING is the target bitrates.  For example: \"23:35 24:3473242 etc...\"." >&2
56  echo "${USAGE_INDENT}$INITIAL_LAMBDA_MODIFIERS_USAGE_STRING is the Lambda-modifiers to use for the first guess.  For example: \"$-{LAMBDA_MODIFIER_PREFIX}23 1e0 $-{LAMBDA_MODIFIER_PREFIX}24 0.98 etc...\"" >&2
57  echo "${USAGE_INDENT}$ENCODE_COMMAND_ARGS_USAGE_STRING is the extra arguments to be passed to encodeCommand.sh.  The common arguments that are available to both $0 and encodeCommand.sh should not be passed though this argument.  For example, don't pass $Q_OPTION here because it is an option of $0.  $EXECUTABLE_OPTION and ($CONFIGURATION_PATH_OPTION or $CONFIGURATION_DIRECTORY_OPTION) must be passed through this argument.  For example, \"$ENCODE_COMMAND_ARGS_OPTION '$EXECUTABLE_OPTION ~/bin/encode.exe $CONFIGURATION_DIRECTORY_OPTION ~/cfg/'\"." >&2
58  echo "${USAGE_INDENT}$EXTRA_ARGUMENTS_USAGE_STRING specifies extra arguments to be passed directly to the encoder (not to encodeCommand.sh)." >&2
59  outputOutputDirectoryUsage
60  outputInputNameUsage
61 
62  exit 1
63}
64
65NORMAL_MODE="NORMAL_MODE"
66RESUME_MODE="RESUME_MODE"
67mode="$NORMAL_MODE"
68
69# For every argument $1
70while [ "" != "$*" ] ; do
71  case $1 in
72    $RESUME_MODE_OPTION)
73      mode="$RESUME_MODE"
74    ;;
75    -*)
76      checkDollarTwo "$1" "$2"
77      case $1 in
78        $EXTRA_ARGUMENTS_OPTION) extraArguments=$2 ;;
79        $Q_OPTION) q=$2 ;;
80        $OUTPUT_DIRECTORY_OPTION) outputDirectory=$2 ;;
81        $CONFIGURATION_IDENTIFIER_OPTION) configurationIdentifier=$2 ;;
82        $TARGET_BITRATES_OPTION) targetBitrates=$2 ;;
83        $INITIAL_LAMBDA_MODIFIERS_OPTION) initialLambdaModifiers=$2 ;;
84        $ENCODE_COMMAND_ARGS_OPTION) encodeCommandArgs=$2 ;;
85        *)
86          printf "You entered an invalid option: \"$1\".\n" >&2
87          outputUsageAndExit
88        ;;
89      esac
90      shift
91    ;;
92    *)
93      if [[ "" == $inputName ]] ; then
94        inputName=$1
95      else
96        printf "You entered too many arguments.\n" >&2
97        outputUsageAndExit
98      fi
99    ;;
100  esac
101 
102  shift
103done
104
105verifyProvided "$Q_STRING" "$q"
106verifyQ $q
107
108verifyProvided "$OUTPUT_DIRECTORY_STRING" "$outputDirectory"
109verifyDirectory "$OUTPUT_DIRECTORY_STRING" "$outputDirectory"
110
111verifyProvided "$CONFIGURATION_IDENTIFIER_STRING" "$configurationIdentifier"
112verifyConfigurationIdentifier "$configurationIdentifier"
113
114verifyProvided "target bitrates ($TARGET_BITRATES_OPTION)" "$targetBitrates"
115verifyProvided "$INPUT_NAME_STRING" "$inputName"
116
117outputPathBegin="${outputDirectory}${inputName}_${configurationIdentifier}_q${q}"
118logPath="$outputPathBegin.log"
119metaLogPath="${outputPathBegin}_meta.log"
120
121TRUE=true
122
123# Outputs "$TRUE" if the given file exist.  Outputs nothing if the given file does not exist.  The first argument is the path to the supposed file.
124function doesFileExist {
125  ls $1 &> /dev/null
126  if [[ 0 == "$?" ]] ; then
127    echo "$TRUE"
128  fi
129}
130
131# Validate the mode (normal or resume) based on whether or not the meta-log file already exists
132if [[ "$TRUE" == "$(doesFileExist "$metaLogPath")" ]] ; then
133  if [[ "$NORMAL_MODE" == "$mode" ]] ; then
134    echo "$metaLogPath already exists.  Consider using resume-mode." >&2
135    outputUsageAndExit
136  else  # Resume-mode
137    cat "$metaLogPath"  # Output the pre-existing meta-log so we can resume where we left off
138  fi
139else  # Meta-log file does not exist
140  if [[ "$RESUME_MODE" == "$mode" ]] ; then
141    echo "$metaLogPath does not exist and resume-mode is enabled." >&2
142    outputUsageAndExit
143  fi
144fi
145
146# Outputs the number of elements in the given bitrate vector
147function bitrateVectorSize {
148  echo "$1" | sed -e 's/[^ ]//g' | wc -c | sed -e 's/^ *//'
149}
150
151# Initialize targetBitrateVectorSize
152targetBitrateVectorSize="$(bitrateVectorSize "$targetBitrates")"
153
154# Outputs the product of the two input values
155function multiply {
156  echo | awk "{print $1 * $2}"
157}
158
159# Extracts the bitrate at the given index from the given bitrate vector.  The first argument is the given index and the second argument in the given bitrate vector.
160function extractBitrateFromVector {
161  local localIndex=$(expr "$1" "+" "1")
162  echo "$2" | awk "{ print \$$localIndex }"
163}
164
165# Outputs a bitrate vector by multiplying the $targetBitrates vector by a given scalar.  The first argument is the given scalar.
166function populateBitrates {
167  local lI=0
168  local lResult=""
169  while true ; do
170    local lTargetBitrate=$(extractBitrateFromVector "$lI" "$targetBitrates")
171    if [[ "" == "$lTargetBitrate" ]] ; then
172      break;
173    fi
174    local lNew=$(multiply "$lTargetBitrate" "$1")
175    lResult="$lResult $lNew"
176    ((++lI))
177  done
178  echo "$lResult" | sed -e 's/^ //'
179}
180
181# Initialize the ranges
182outerRangeMins=$(populateBitrates "0.980")
183innerRangeMins=$(populateBitrates "0.985")
184innerRangeMaxs=$(populateBitrates "1.015")
185outerRangeMaxs=$(populateBitrates "1.020")
186
187# Outputs the given string to the meta-log (both the file and stdout) with no newline character.  The first argument is the string to output.
188function outputToMetaLogNoNewline {
189  toPrint=`echo $1 | sed -e 's/%/%%/g'`
190  printf -- "$toPrint"
191  printf -- "$toPrint" >> $metaLogPath
192}
193
194# Outputs the given string to the meta-log (both the file and stdout) with a newline character.  The first argument is the string to output.
195function outputToMetaLogWithNewline {
196  echo "$1"
197  echo "$1" >> $metaLogPath
198}
199
200# Extracts the Lambda-modifier at the given index from $lambdaModifiers.  The first argument is the given index.
201function extractLambdaModifier {
202  printf -- "$lambdaModifiers" | sed -e 's/^-//' | sed -e 's/ -/\
203/g' | grep "${LAMBDA_MODIFIER_PREFIX}$1 " | sed -e 's/^[^ ]* //'
204}
205
206# Outputs the given Lambda-modifier with a fixed number of decimal points.  The first argument is the given Lambda-modifier.
207function formatLambdaModifier {
208  printf "%.7f" "$1"
209}
210
211# Outputs $lambdaModifiers to the meta-log with proper formatting
212function outputLambdaModifiersToMetaLog {
213  local lI=0
214  local lLambdaModifier=$(extractLambdaModifier "$lI")
215  local lLambdaModifier=$(formatLambdaModifier "$lLambdaModifier")
216  local lOutput="-${LAMBDA_MODIFIER_PREFIX}$lI $lLambdaModifier"
217  while true ; do
218    ((++lI))
219    local lLambdaModifier=$(extractLambdaModifier "$lI")
220    if [[ "" == "$lLambdaModifier" ]] ; then
221      break
222    fi
223    local lLambdaModifier=$(formatLambdaModifier "$lLambdaModifier")
224    local lOutput="$lOutput -${LAMBDA_MODIFIER_PREFIX}$lI $lLambdaModifier"
225  done
226  outputToMetaLogNoNewline "$lOutput;"
227}
228
229# Initialize lambdaModifiers and output it to the meta-log
230if [[ "$RESUME_MODE" == "$mode" ]] ; then
231  if [[ "" == "$initialLambdaModifiers" ]] ; then  # If no initial lambda-modifiers provided, use default value
232    lambdaModifiers=$(tail -n 1 < "$metaLogPath" | sed -e 's/;$//')
233  else  # Initial lambda-modifiers provided
234    echo "You cannot use $RESUME_MODE and specify the initial lambda-modifiers.  In resume-mode, the lambda-modifiers will be retreived from the last line of the meta-log." >&2
235    outputUsageAndExit
236  fi
237else
238  if [[ "" == "$initialLambdaModifiers" ]] ; then  # If no initial lambda-modifiers provided, use default value
239    lambdaModifiers="-${LAMBDA_MODIFIER_PREFIX}0 1"
240    for (( i=1; i<"$targetBitrateVectorSize"; ++i )); do
241      lambdaModifiers="$lambdaModifiers -${LAMBDA_MODIFIER_PREFIX}${i} 1"
242    done
243  else  # Initial lambda-modifiers provided
244    lambdaModifiers="$initialLambdaModifiers"
245  fi
246  outputLambdaModifiersToMetaLog
247fi
248
249# Calculates the difference percentage between the given target bitrate and the given bitrate, appropriately formats this difference percentage, and then outputs it.  The first argument is the given target bitrate and the second argument is the given bitrate.
250function calculateAndFormatDifferencePercentage {
251  # Calculate the result and format it with the right number of decimal places
252  local result=$(echo | awk "{print 100*($2-$1)/$1}")
253  local result=$(printf "%.3f" "$result")
254 
255  # Separate the sign from the result
256  local sign=$(echo "$result" | sed -e 's/[^-]*$//')
257  if [[ "$sign" != "-" ]] ; then
258    local sign="+"
259  fi
260  local result=$(echo "$result" | sed -e 's/^-//')
261 
262  # Pad leading zereos to make two digits before the decimal point
263  if [[ 2 == $(echo "$result" | sed -e 's/\..*$//' | wc -c | sed -e 's/^ *//') ]] ; then
264    local result="0$result"
265  fi
266 
267  # Output the result including the sign and the percent sign
268  echo "${sign}${result}%"
269}
270
271# Outputs $TRUE i.f.f. the first argument is less than the second argument
272function lessOrEqual {
273  echo | awk "{ if($1 < $2) print \"$TRUE\" }"
274}
275
276# Outputs $TRUE i.f.f. the second argument is greater than the first argument and less than the third argument ($1 < $2 < $3)
277function isInRange {
278  if [[ "$TRUE" == $(lessOrEqual "$1" "$2") ]] ; then
279    if [[ "$TRUE" == $(lessOrEqual "$2" "$3") ]] ; then
280      echo "$TRUE"
281    fi
282  fi
283}
284
285# From the given bitrate vector, outputs the "bad" bitrates by filtering out the "good" bitrates.  The first argument is the index of the last good bitrate and the second argument is the given bitrate vector.  If the first argument is -1, then the given bitrate vector is outputted in its entirety.
286function filterOutGoodBitrates {
287  local result="$2"
288  for (( i=0; i<="$1"; ++i )); do
289    result=$(echo "$result" | sed -e 's/^[^ ]* //')
290  done
291  echo "$result"
292}
293
294# Outputs a given line from the given variable.  The first argument is the line number to output and the second argument is the given variable to extract the line from.
295function outputLine {
296  echo "$2" | head -n "$(expr "$1" + 1)" | tail -n 1
297}
298
299# Initialize iterationCount
300if [[ "$RESUME_MODE" == "$mode" ]] ; then
301  iterationCount=$(wc -l < "$metaLogPath" | sed -e 's/^ *//')
302else
303  iterationCount=0
304fi
305
306ITERATION_COUNT_LIMIT=50  # The number of attempts to make before giving up
307
308while true ; do  # The main loop
309 
310  # Run the encoder
311  sh ${SUB_TOOLS_DIRECTORY}encodeCommand.sh $inputName $encodeCommandArgs $CONFIGURATION_IDENTIFIER_OPTION $configurationIdentifier $Q_OPTION $q $OUTPUT_DIRECTORY_OPTION $outputDirectory -ea "$extraArguments $lambdaModifiers" | sh > $logPath
312  if [[ $? != 0 ]] ; then
313    printf "Unexpected exit status from encodeCommand.sh\n" >&2
314    exit 1
315  fi
316 
317  # Extract and output the bitrates
318  bitrates=`${SUB_TOOLS_DIRECTORY}extractBitrates.exe < $logPath`
319  outputToMetaLogNoNewline "$bitrates;"
320 
321  # Make sure that the index set of the extracted bitrates matches the index set of the target bitrates
322  if [[ "$targetBitrateVectorSize" != "$(bitrateVectorSize "$bitrates")" ]] ; then
323    echo "Index set from the extracted bitrates does not match the index set from the target bitrates" >&2
324    exit 1
325  fi
326 
327  # Calculate the bitrate difference percentages and output them to the meta-log
328  percentages="$(calculateAndFormatDifferencePercentage "$(extractBitrateFromVector "0" "$targetBitrates")" "$(extractBitrateFromVector "0" "$bitrates")")"
329  for (( i=1; i<"$targetBitrateVectorSize"; ++i )); do
330    percentages="$percentages $(calculateAndFormatDifferencePercentage "$(extractBitrateFromVector "$i" "$targetBitrates")" "$(extractBitrateFromVector "$i" "$bitrates")")"
331  done
332  outputToMetaLogNoNewline "$percentages;"
333 
334  # Initialize and output areBitratesSatismodifiery
335  areBitratesSatisfactory=yes
336  for (( i=0; ; ++i )) ; do
337    outerRangeMin=$(extractBitrateFromVector "$i" "$outerRangeMins")
338    bitrate=$(extractBitrateFromVector "$i" "$bitrates")
339    outerRangeMax=$(extractBitrateFromVector "$i" "$outerRangeMaxs")
340    if [[ "" == "$bitrate" ]] ; then
341      break
342    fi
343    if [[ $(isInRange "$outerRangeMin" "$bitrate" "$outerRangeMax") != "$TRUE" ]] ; then
344      areBitratesSatisfactory=no
345      break
346    fi
347  done
348  outputToMetaLogWithNewline "$areBitratesSatisfactory"
349 
350  # Exit if we are finished or if we have iterated too many times
351  if [[ yes == $areBitratesSatisfactory ]] ; then
352    mv "$logPath" "${outputPathBegin}_final.log"
353    exit 0
354  else
355    # Rename the deprecated log
356    countString="$iterationCount"
357    if [[ 1 == `printf -- "$countString" | wc -c | sed -e 's/^ *//'` ]] ; then
358      countString="0$countString"
359    fi
360    mv "$logPath" "${outputPathBegin}_dep${countString}.log"
361   
362    ((++iterationCount))
363    if [[ "$ITERATION_COUNT_LIMIT" == "$iterationCount" ]] ; then
364      outputToMetaLogWithNewline "Could not reach target bitrates"
365      exit 1
366    fi
367  fi
368 
369  filteredMetaLog=$(sed -e 's/;[^;]*$//' < $metaLogPath | sed -e 's/;[^;]*$//')
370  bitratesFromMetaLog=$(printf -- "$filteredMetaLog" | sed -e 's/^[^;]*;//')
371 
372  # Initialize goodIndex
373  goodIndex=-1
374  for (( i=0; i<"$targetBitrateVectorSize"; ++i )); do
375    innerRangeMin=$(extractBitrateFromVector "$i" "$innerRangeMins")
376    bitrate=$(extractBitrateFromVector "$i" "$bitrates")
377    innerRangeMax=$(extractBitrateFromVector "$i" "$innerRangeMaxs")
378    if [[ "$TRUE" == $(isInRange "$innerRangeMin" "$bitrate" "$innerRangeMax") ]] ; then
379      goodIndex="$i"
380    else
381      break
382    fi
383  done
384 
385  badBitrates=$(filterOutGoodBitrates "$goodIndex" "$bitratesFromMetaLog")
386  lambdaModifiersFromMetaLog=`printf -- "$filteredMetaLog" | sed -e 's/;[^;]*$//'`
387  badLambdaModifiers=`printf -- "$lambdaModifiersFromMetaLog" | sed -e "s/^.*-${LAMBDA_MODIFIER_PREFIX}$goodIndex [^ ]* //"`
388  lineCount=`printf -- "$badBitrates\n" | wc -l | sed -e 's/^ *//'`
389 
390  # Initialize guessLambdaModifiersIn
391  guessLambdaModifiersIn="$(outputLine 0 "$badLambdaModifiers");$(outputLine 0 "$badBitrates")"
392  for (( i=1; i<"$lineCount"; ++i )); do
393    guessLambdaModifiersIn="$(printf -- "$guessLambdaModifiersIn\n$(outputLine "$i" "$badLambdaModifiers");$(outputLine "$i" "$badBitrates")")"
394  done
395 
396  # Run guessLambdaModifiers
397  guessedLambdaModifiers=$(printf -- "$guessLambdaModifiersIn" | ${SUB_TOOLS_DIRECTORY}guessLambdaModifiers.exe "-.5" "$(filterOutGoodBitrates "$goodIndex" "$targetBitrates")")
398  if [[ $? != 0 ]] ; then
399    printf "Unexpected exit status from guessLambdaModifiers.exe\n" >&2
400    exit 1
401  fi
402 
403  # Initialize lambdaModifiers and output them to the meta-log
404  lastLambdaModifiersFromMetaLog=`printf -- "$filteredMetaLog" | tail -n 1 | sed -e "s/;[^;]*$//"`
405  goodLambdaModifiersFromMetaLog=`printf -- "$lastLambdaModifiersFromMetaLog" | tail -n 1 | sed -e "s/-${LAMBDA_MODIFIER_PREFIX}$(expr "$goodIndex" + 1).*$//"`
406  lambdaModifiers="${goodLambdaModifiersFromMetaLog}${guessedLambdaModifiers}"
407  outputLambdaModifiersToMetaLog
408 
409done
Note: See TracBrowser for help on using the repository browser.