0001 #!/usr/bin/env bash
0002
0003 #
0004 # Licensed to the Apache Software Foundation (ASF) under one or more
0005 # contributor license agreements. See the NOTICE file distributed with
0006 # this work for additional information regarding copyright ownership.
0007 # The ASF licenses this file to You under the Apache License, Version 2.0
0008 # (the "License"); you may not use this file except in compliance with
0009 # the License. You may obtain a copy of the License at
0010 #
0011 # http://www.apache.org/licenses/LICENSE-2.0
0012 #
0013 # Unless required by applicable law or agreed to in writing, software
0014 # distributed under the License is distributed on an "AS IS" BASIS,
0015 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0016 # See the License for the specific language governing permissions and
0017 # limitations under the License.
0018 #
0019
0020 # Stop on error
0021 set -e
0022 # Set nullglob for when we are checking existence based on globs
0023 shopt -s nullglob
0024
0025 FWDIR="$(cd "$(dirname "$0")"/..; pwd)"
0026 cd "$FWDIR"
0027
0028 echo "Constructing virtual env for testing"
0029 VIRTUALENV_BASE=$(mktemp -d)
0030
0031 # Clean up the virtual env environment used if we created one.
0032 function delete_virtualenv() {
0033 echo "Cleaning up temporary directory - $VIRTUALENV_BASE"
0034 rm -rf "$VIRTUALENV_BASE"
0035 }
0036 trap delete_virtualenv EXIT
0037
0038 PYTHON_EXECS=()
0039 # Some systems don't have pip or virtualenv - in those cases our tests won't work.
0040 if hash virtualenv 2>/dev/null && [ ! -n "$USE_CONDA" ]; then
0041 echo "virtualenv installed - using. Note if this is a conda virtual env you may wish to set USE_CONDA"
0042 # test only against python3
0043 if hash python3 2>/dev/null; then
0044 PYTHON_EXECS=('python3')
0045 else
0046 echo "Python3 not installed on system, skipping pip installability tests"
0047 exit 0
0048 fi
0049 elif hash conda 2>/dev/null; then
0050 echo "Using conda virtual environments"
0051 PYTHON_EXECS=('3.6')
0052 USE_CONDA=1
0053 else
0054 echo "Missing virtualenv & conda, skipping pip installability tests"
0055 exit 0
0056 fi
0057 if ! hash pip 2>/dev/null; then
0058 echo "Missing pip, skipping pip installability tests."
0059 exit 0
0060 fi
0061
0062 # Determine which version of PySpark we are building for archive name
0063 PYSPARK_VERSION=$(python3 -c "exec(open('python/pyspark/version.py').read());print(__version__)")
0064 PYSPARK_DIST="$FWDIR/python/dist/pyspark-$PYSPARK_VERSION.tar.gz"
0065 # The pip install options we use for all the pip commands
0066 PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall "
0067 # Test both regular user and edit/dev install modes.
0068 PIP_COMMANDS=("pip install $PIP_OPTIONS $PYSPARK_DIST"
0069 "pip install $PIP_OPTIONS -e python/")
0070
0071 for python in "${PYTHON_EXECS[@]}"; do
0072 for install_command in "${PIP_COMMANDS[@]}"; do
0073 echo "Testing pip installation with python $python"
0074 # Create a temp directory for us to work in and save its name to a file for cleanup
0075 echo "Using $VIRTUALENV_BASE for virtualenv"
0076 VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python
0077 rm -rf "$VIRTUALENV_PATH"
0078 if [ -n "$USE_CONDA" ]; then
0079 conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools
0080 source activate "$VIRTUALENV_PATH"
0081 else
0082 mkdir -p "$VIRTUALENV_PATH"
0083 virtualenv --python=$python "$VIRTUALENV_PATH"
0084 source "$VIRTUALENV_PATH"/bin/activate
0085 fi
0086 # Upgrade pip & friends if using virtual env
0087 if [ ! -n "$USE_CONDA" ]; then
0088 pip install --upgrade pip wheel numpy
0089 fi
0090
0091 echo "Creating pip installable source dist"
0092 cd "$FWDIR"/python
0093 # Delete the egg info file if it exists, this can cache the setup file.
0094 rm -rf pyspark.egg-info || echo "No existing egg info file, skipping deletion"
0095 python3 setup.py sdist
0096
0097
0098 echo "Installing dist into virtual env"
0099 cd dist
0100 # Verify that the dist directory only contains one thing to install
0101 sdists=(*.tar.gz)
0102 if [ ${#sdists[@]} -ne 1 ]; then
0103 echo "Unexpected number of targets found in dist directory - please cleanup existing sdists first."
0104 exit -1
0105 fi
0106 # Do the actual installation
0107 cd "$FWDIR"
0108 $install_command
0109
0110 cd /
0111
0112 echo "Run basic sanity check on pip installed version with spark-submit"
0113 spark-submit "$FWDIR"/dev/pip-sanity-check.py
0114 echo "Run basic sanity check with import based"
0115 python3 "$FWDIR"/dev/pip-sanity-check.py
0116 echo "Run the tests for context.py"
0117 python3 "$FWDIR"/python/pyspark/context.py
0118
0119 cd "$FWDIR"
0120
0121 # conda / virtualenv environments need to be deactivated differently
0122 if [ -n "$USE_CONDA" ]; then
0123 source deactivate
0124 else
0125 deactivate
0126 fi
0127
0128 done
0129 done
0130
0131 exit 0
