0001 #!/usr/bin/env bash
0002 #
0003 # Licensed to the Apache Software Foundation (ASF) under one or more
0004 # contributor license agreements. See the NOTICE file distributed with
0005 # this work for additional information regarding copyright ownership.
0006 # The ASF licenses this file to You under the Apache License, Version 2.0
0007 # (the "License"); you may not use this file except in compliance with
0008 # the License. You may obtain a copy of the License at
0009 #
0010 # http://www.apache.org/licenses/LICENSE-2.0
0011 #
0012 # Unless required by applicable law or agreed to in writing, software
0013 # distributed under the License is distributed on an "AS IS" BASIS,
0014 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0015 # See the License for the specific language governing permissions and
0016 # limitations under the License.
0017 #
0018 # define test binaries + versions
0019 FLAKE8_BUILD="flake8"
0020 MINIMUM_FLAKE8="3.5.0"
0021
0022 PYCODESTYLE_BUILD="pycodestyle"
0023 MINIMUM_PYCODESTYLE="2.4.0"
0024
0025 SPHINX_BUILD="sphinx-build"
0026
0027 PYTHON_EXECUTABLE="python3"
0028
0029 function satisfies_min_version {
0030 local provided_version="$1"
0031 local expected_version="$2"
0032 echo "$(
0033 "$PYTHON_EXECUTABLE" << EOM
0034 from setuptools.extern.packaging import version
0035 print(version.parse('$provided_version') >= version.parse('$expected_version'))
0036 EOM
0037 )"
0038 }
0039
0040 function compile_python_test {
0041 local COMPILE_STATUS=
0042 local COMPILE_REPORT=
0043
0044 if [[ ! "$1" ]]; then
0045 echo "No python files found! Something is very wrong -- exiting."
0046 exit 1;
0047 fi
0048
0049 # compileall: https://docs.python.org/3/library/compileall.html
0050 echo "starting python compilation test..."
0051 COMPILE_REPORT=$( (python3 -B -mcompileall -q -l -x "[/\\\\][.]git" $1) 2>&1)
0052 COMPILE_STATUS=$?
0053
0054 if [ $COMPILE_STATUS -ne 0 ]; then
0055 echo "Python compilation failed with the following errors:"
0056 echo "$COMPILE_REPORT"
0057 echo "$COMPILE_STATUS"
0058 exit "$COMPILE_STATUS"
0059 else
0060 echo "python compilation succeeded."
0061 echo
0062 fi
0063 }
0064
0065 function pycodestyle_test {
0066 local PYCODESTYLE_STATUS=
0067 local PYCODESTYLE_REPORT=
0068 local RUN_LOCAL_PYCODESTYLE=
0069 local PYCODESTYLE_VERSION=
0070 local EXPECTED_PYCODESTYLE=
0071 local PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py"
0072 local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py"
0073
0074 if [[ ! "$1" ]]; then
0075 echo "No python files found! Something is very wrong -- exiting."
0076 exit 1;
0077 fi
0078
0079 # check for locally installed pycodestyle & version
0080 RUN_LOCAL_PYCODESTYLE="False"
0081 if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then
0082 PYCODESTYLE_VERSION="$($PYCODESTYLE_BUILD --version)"
0083 EXPECTED_PYCODESTYLE="$(satisfies_min_version $PYCODESTYLE_VERSION $MINIMUM_PYCODESTYLE)"
0084 if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then
0085 RUN_LOCAL_PYCODESTYLE="True"
0086 fi
0087 fi
0088
0089 # download the right version or run locally
0090 if [ $RUN_LOCAL_PYCODESTYLE == "False" ]; then
0091 # Get pycodestyle at runtime so that we don't rely on it being installed on the build server.
0092 # See: https://github.com/apache/spark/pull/1744#issuecomment-50982162
0093 # Updated to the latest official version of pep8. pep8 is formally renamed to pycodestyle.
0094 echo "downloading pycodestyle from $PYCODESTYLE_SCRIPT_REMOTE_PATH..."
0095 if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then
0096 curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH"
0097 local curl_status="$?"
0098
0099 if [ "$curl_status" -ne 0 ]; then
0100 echo "Failed to download pycodestyle.py from $PYCODESTYLE_SCRIPT_REMOTE_PATH"
0101 exit "$curl_status"
0102 fi
0103 fi
0104
0105 echo "starting pycodestyle test..."
0106 PYCODESTYLE_REPORT=$( (python3 "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $1) 2>&1)
0107 PYCODESTYLE_STATUS=$?
0108 else
0109 # we have the right version installed, so run locally
0110 echo "starting pycodestyle test..."
0111 PYCODESTYLE_REPORT=$( ($PYCODESTYLE_BUILD --config=dev/tox.ini $1) 2>&1)
0112 PYCODESTYLE_STATUS=$?
0113 fi
0114
0115 if [ $PYCODESTYLE_STATUS -ne 0 ]; then
0116 echo "pycodestyle checks failed:"
0117 echo "$PYCODESTYLE_REPORT"
0118 exit "$PYCODESTYLE_STATUS"
0119 else
0120 echo "pycodestyle checks passed."
0121 echo
0122 fi
0123 }
0124
0125 function flake8_test {
0126 local FLAKE8_VERSION=
0127 local EXPECTED_FLAKE8=
0128 local FLAKE8_REPORT=
0129 local FLAKE8_STATUS=
0130
0131 if ! hash "$FLAKE8_BUILD" 2> /dev/null; then
0132 echo "The flake8 command was not found."
0133 echo "flake8 checks failed."
0134 exit 1
0135 fi
0136
0137 _FLAKE8_VERSION=($($FLAKE8_BUILD --version))
0138 FLAKE8_VERSION="${_FLAKE8_VERSION[0]}"
0139 EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)"
0140
0141 if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
0142 echo "\
0143 The minimum flake8 version needs to be $MINIMUM_FLAKE8. Your current version is $FLAKE8_VERSION
0144
0145 flake8 checks failed."
0146 exit 1
0147 fi
0148
0149 echo "starting $FLAKE8_BUILD test..."
0150 FLAKE8_REPORT=$( ($FLAKE8_BUILD . --count --select=E901,E999,F821,F822,F823 \
0151 --max-line-length=100 --show-source --statistics) 2>&1)
0152 FLAKE8_STATUS=$?
0153
0154 if [ "$FLAKE8_STATUS" -ne 0 ]; then
0155 echo "flake8 checks failed:"
0156 echo "$FLAKE8_REPORT"
0157 echo "$FLAKE8_STATUS"
0158 exit "$FLAKE8_STATUS"
0159 else
0160 echo "flake8 checks passed."
0161 echo
0162 fi
0163 }
0164
0165 function sphinx_test {
0166 local SPHINX_REPORT=
0167 local SPHINX_STATUS=
0168
0169 # Check that the documentation builds acceptably, skip check if sphinx is not installed.
0170 if ! hash "$SPHINX_BUILD" 2> /dev/null; then
0171 echo "The $SPHINX_BUILD command was not found. Skipping pydoc checks for now."
0172 echo
0173 return
0174 fi
0175
0176 echo "starting $SPHINX_BUILD tests..."
0177 pushd python/docs &> /dev/null
0178 make clean &> /dev/null
0179 # Treat warnings as errors so we stop correctly
0180 SPHINX_REPORT=$( (SPHINXOPTS="-a -W" make html) 2>&1)
0181 SPHINX_STATUS=$?
0182
0183 if [ "$SPHINX_STATUS" -ne 0 ]; then
0184 echo "$SPHINX_BUILD checks failed:"
0185 echo "$SPHINX_REPORT"
0186 echo
0187 echo "re-running make html to print full warning list:"
0188 make clean &> /dev/null
0189 SPHINX_REPORT=$( (SPHINXOPTS="-a" make html) 2>&1)
0190 echo "$SPHINX_REPORT"
0191 exit "$SPHINX_STATUS"
0192 else
0193 echo "$SPHINX_BUILD checks passed."
0194 echo
0195 fi
0196
0197 popd &> /dev/null
0198 }
0199
0200 SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
0201 SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
0202
0203 pushd "$SPARK_ROOT_DIR" &> /dev/null
0204
0205 PYTHON_SOURCE="$(find . -name "*.py")"
0206
0207 compile_python_test "$PYTHON_SOURCE"
0208 pycodestyle_test "$PYTHON_SOURCE"
0209 flake8_test
0210 sphinx_test
0211
0212 echo
0213 echo "all lint-python tests passed!"
0214
0215 popd &> /dev/null
