0001 #!/usr/bin/env bash
0002
0003 #
0004 # Licensed to the Apache Software Foundation (ASF) under one or more
0005 # contributor license agreements. See the NOTICE file distributed with
0006 # this work for additional information regarding copyright ownership.
0007 # The ASF licenses this file to You under the Apache License, Version 2.0
0008 # (the "License"); you may not use this file except in compliance with
0009 # the License. You may obtain a copy of the License at
0010 #
0011 # http://www.apache.org/licenses/LICENSE-2.0
0012 #
0013 # Unless required by applicable law or agreed to in writing, software
0014 # distributed under the License is distributed on an "AS IS" BASIS,
0015 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0016 # See the License for the specific language governing permissions and
0017 # limitations under the License.
0018 #
0019
0020 if [ -z "${SPARK_HOME}" ]; then
0021 source "$(dirname "$0")"/find-spark-home
0022 fi
0023
0024 source "${SPARK_HOME}"/bin/load-spark-env.sh
0025 export _SPARK_CMD_USAGE="Usage: ./bin/pyspark [options]"
0026
0027 # In Spark 2.0, IPYTHON and IPYTHON_OPTS are removed and pyspark fails to launch if either option
0028 # is set in the user's environment. Instead, users should set PYSPARK_DRIVER_PYTHON=ipython
0029 # to use IPython and set PYSPARK_DRIVER_PYTHON_OPTS to pass options when starting the Python driver
0030 # (e.g. PYSPARK_DRIVER_PYTHON_OPTS='notebook'). This supports full customization of the IPython
0031 # and executor Python executables.
0032
0033 # Fail noisily if removed options are set
0034 if [[ -n "$IPYTHON" || -n "$IPYTHON_OPTS" ]]; then
0035 echo "Error in pyspark startup:"
0036 echo "IPYTHON and IPYTHON_OPTS are removed in Spark 2.0+. Remove these from the environment and set PYSPARK_DRIVER_PYTHON and PYSPARK_DRIVER_PYTHON_OPTS instead."
0037 exit 1
0038 fi
0039
0040 # Default to standard python interpreter unless told otherwise
0041 if [[ -z "$PYSPARK_PYTHON" ]]; then
0042 PYSPARK_PYTHON=python
0043 fi
0044 if [[ -z "$PYSPARK_DRIVER_PYTHON" ]]; then
0045 PYSPARK_DRIVER_PYTHON=$PYSPARK_PYTHON
0046 fi
0047 export PYSPARK_PYTHON
0048 export PYSPARK_DRIVER_PYTHON
0049 export PYSPARK_DRIVER_PYTHON_OPTS
0050
0051 # Add the PySpark classes to the Python path:
0052 export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH"
0053 export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip:$PYTHONPATH"
0054
0055 # Load the PySpark shell.py script when ./pyspark is used interactively:
0056 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
0057 export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py"
0058
0059 # For pyspark tests
0060 if [[ -n "$SPARK_TESTING" ]]; then
0061 unset YARN_CONF_DIR
0062 unset HADOOP_CONF_DIR
0063 export PYTHONHASHSEED=0
0064 exec "$PYSPARK_DRIVER_PYTHON" -m "$@"
0065 exit
0066 fi
0067
0068 exec "${SPARK_HOME}"/bin/spark-submit pyspark-shell-main --name "PySparkShell" "$@"
