tools/valgrind/waterfall.sh

#!/bin/bash

# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This script can be used by waterfall sheriffs to fetch the status
# of Valgrind bots on the memory waterfall and test if their local
# suppressions match the reports on the waterfall.

set -e

THISDIR=$(dirname "${0}")
LOGS_DIR=$THISDIR/waterfall.tmp
WATERFALL_PAGE="https://build.chromium.org/p/chromium.memory/builders"
WATERFALL_FYI_PAGE="https://build.chromium.org/p/chromium.memory.fyi/builders"

download() {
  # Download a file.
  # $1 = URL to download
  # $2 = Path to the output file
  # {{{1
  if [ "$(which curl)" != "" ]
  then
    if ! curl -s -o "$2" "$1"
    then
      echo
      echo "Failed to download '$1'... aborting"
      exit 1
    fi
  elif [ "$(which wget)" != "" ]
  then
    if ! wget "$1" -O "$2" -q
    then
      echo
      echo "Failed to download '$1'... aborting"
      exit 1
    fi
  else
    echo "Need either curl or wget to download stuff... aborting"
    exit 1
  fi
  # }}}
}

fetch_logs() {
  # Fetch Valgrind logs from the waterfall {{{1

  # TODO(timurrrr,maruel): use JSON, see
  # http://build.chromium.org/p/chromium.memory/json/help

  rm -rf "$LOGS_DIR" # Delete old logs
  mkdir "$LOGS_DIR"

  URL=$1
  LAYOUT_ONLY=0
  if [ $# != 0 ]; then
    shift
    if [ "$1" = "layout_only" ]; then
      LAYOUT_ONLY=1
    fi
  fi

  echo "Fetching the list of builders..."
  download "$URL" "$LOGS_DIR/builders"
  SLAVES=$(grep "<a href=\"builders\/" "$LOGS_DIR/builders" | \
           grep 'td class="box"' | \
           sed "s/.*<a href=\"builders\///" | sed "s/\".*//" | \
           sort | uniq)

  for S in $SLAVES
  do
    if [ "$LAYOUT_ONLY" = "1" ]; then
      if [ "$S" != "Webkit%20Linux%20%28valgrind%20layout%29" ]; then
        continue;
      fi
    fi

    SLAVE_URL="$URL/$S"
    SLAVE_NAME=$(echo $S | sed -e "s/%20/ /g" -e "s/%28/(/g" -e "s/%29/)/g")
    echo -n "Fetching builds by slave '${SLAVE_NAME}'"
    download $SLAVE_URL?numbuilds=${NUMBUILDS} "$LOGS_DIR/slave_${S}"

    # We speed up the 'fetch' step by skipping the builds/tests which succeeded.
    # TODO(timurrrr): OTOH, we won't be able to check
    # if some suppression is not used anymore.
    #
    # The awk script here joins the lines ending with </td> to make it possible
    # to find the failed builds.
    LIST_OF_BUILDS=$(cat "$LOGS_DIR/slave_$S" | \
                     awk 'BEGIN { buf = "" }
                          {
                            if ($0 ~ /<\/td>/) { buf = (buf $0); }
                            else {
                              if (buf) { print buf; buf="" }
                              print $0
                            }
                          }
                          END {if (buf) print buf}' | \
                     grep "success\|failure" | \
                     head -n $NUMBUILDS | \
                     grep "failure" | \
                     grep -v "failed compile" | \
                     sed "s/.*\/builds\///" | sed "s/\".*//")

    for BUILD in $LIST_OF_BUILDS
    do
      # We'll fetch a few tiny URLs now, let's use a temp file.
      TMPFILE=$(mktemp -t memory_waterfall.XXXXXX)
      download $SLAVE_URL/builds/$BUILD "$TMPFILE"

      REPORT_FILE="$LOGS_DIR/report_${S}_${BUILD}"
      rm -f $REPORT_FILE 2>/dev/null || true  # make sure it doesn't exist

      REPORT_URLS=$(grep -o "[0-9]\+/steps/memory.*/logs/[0-9A-F]\{16\}" \
                    "$TMPFILE" \
                    || true)  # `true` is to succeed on empty output
      FAILED_TESTS=$(grep -o "[0-9]\+/steps/memory.*/logs/[A-Za-z0-9_.]\+" \
                     "$TMPFILE" | grep -v "[0-9A-F]\{16\}" \
                     | grep -v "stdio" || true)

      for REPORT in $REPORT_URLS
      do
        download "$SLAVE_URL/builds/$REPORT/text" "$TMPFILE"
        echo "" >> "$TMPFILE"  # Add a newline at the end
        cat "$TMPFILE" | tr -d '\r' >> "$REPORT_FILE"
      done

      for FAILURE in $FAILED_TESTS
      do
        echo -n "FAILED:" >> "$REPORT_FILE"
        echo "$FAILURE" | sed -e "s/.*\/logs\///" -e "s/\/.*//" \
          >> "$REPORT_FILE"
      done

      rm "$TMPFILE"
      echo $SLAVE_URL/builds/$BUILD >> "$REPORT_FILE"
    done
    echo " DONE"
  done
  # }}}
}

match_suppressions() {
  PYTHONPATH=$THISDIR/../python/google \
             python "$THISDIR/test_suppressions.py" $@ "$LOGS_DIR/report_"*
}

match_gtest_excludes() {
  for PLATFORM in "Linux" "Chromium%20Mac" "Chromium%20OS" "Windows"
  do
    echo
    echo "Test failures on ${PLATFORM}:" | sed "s/%20/ /"
    grep -h -o "^FAILED:.*" -R "$LOGS_DIR"/*${PLATFORM}* | \
         grep -v "FAILS\|FLAKY" | sort | uniq | \
         sed -e "s/^FAILED://" -e "s/^/  /"
    # Don't put any operators between "grep | sed" and "RESULT=$PIPESTATUS"
    RESULT=$PIPESTATUS

    if [ "$RESULT" == 1 ]
    then
      echo "  None!"
    else
      echo
      echo "  Note: we don't check for failures already excluded locally yet"
      echo "  TODO(timurrrr): don't list tests we've already excluded locally"
    fi
  done
  echo
  echo "Note: we don't print FAILS/FLAKY tests and 1200s-timeout failures"
}

usage() {
  cat <<EOF
usage: $0 fetch|match options

This script can be used by waterfall sheriffs to fetch the status
of Valgrind bots on the memory waterfall and test if their local
suppressions match the reports on the waterfall.

OPTIONS:
   -h      Show this message
   -n N    Fetch N builds from each slave.

COMMANDS:
  fetch           Fetch Valgrind logs from the memory waterfall
  fetch_layout    Fetch many Valgrind logs from the layout test bot only
  match           Test the local suppression files against the downloaded logs

EOF
}

CMD=$1
if [ $# != 0 ]; then
  shift
fi

NUMBUILDS=3
if [ "$CMD" = "fetch_layout" ]; then
  NUMBUILDS=30
fi

# Arguments for "match" are handled in match_suppressions
if [ "$CMD" != "match" ]; then
  while getopts “hn:” OPTION
  do
    case $OPTION in
      h)
        usage
        exit
        ;;
      n)
        NUMBUILDS=$OPTARG
        ;;
      ?)
        usage
        exit
        ;;
    esac
  done
  shift $((OPTIND-1))
  if [ $# != 0 ]; then
    usage
    exit 1
  fi
fi

if [ "$CMD" = "fetch" ]; then
  echo "Fetching $NUMBUILDS builds"
  fetch_logs "$WATERFALL_PAGE"
  fetch_logs "$WATERFALL_FYI_PAGE"
elif [ "$CMD" = "fetch_layout" ]; then
  echo "Fetching $NUMBUILDS builds"
  fetch_logs "$WATERFALL_FYI_PAGE" layout_only
elif [ "$CMD" = "match" ]; then
  match_suppressions $@
  match_gtest_excludes
elif [ "$CMD" = "blame" ]; then
  echo The blame command died of bitrot. If you need it, please reimplement it.
  echo Reimplementation is blocked on http://crbug.com/82688
else
  usage
  exit 1
fi