dockerfiles/spark/entrypoint.sh

0001 #!/bin/bash
0002 #
0003 # Licensed to the Apache Software Foundation (ASF) under one or more
0004 # contributor license agreements.  See the NOTICE file distributed with
0005 # this work for additional information regarding copyright ownership.
0006 # The ASF licenses this file to You under the Apache License, Version 2.0
0007 # (the "License"); you may not use this file except in compliance with
0008 # the License.  You may obtain a copy of the License at
0009 #
0010 #    http://www.apache.org/licenses/LICENSE-2.0
0011 #
0012 # Unless required by applicable law or agreed to in writing, software
0013 # distributed under the License is distributed on an "AS IS" BASIS,
0014 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0015 # See the License for the specific language governing permissions and
0016 # limitations under the License.
0017 #
0018
0019 # echo commands to the terminal output
0020 set -ex
0021
0022 # Check whether there is a passwd entry for the container UID
0023 myuid=$(id -u)
0024 mygid=$(id -g)
0025 # turn off -e for getent because it will return error code in anonymous uid case
0026 set +e
0027 uidentry=$(getent passwd $myuid)
0028 set -e
0029
0030 # If there is no passwd entry for the container UID, attempt to create one
0031 if [ -z "$uidentry" ] ; then
0032     if [ -w /etc/passwd ] ; then
0033         echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
0034     else
0035         echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
0036     fi
0037 fi
0038
0039 SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
0040 env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
0041 readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
0042
0043 if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
0044   SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
0045 fi
0046
0047 if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "2" ]; then
0048     pyv="$(python -V 2>&1)"
0049     export PYTHON_VERSION="${pyv:7}"
0050     export PYSPARK_PYTHON="python"
0051     export PYSPARK_DRIVER_PYTHON="python"
0052 elif [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then
0053     pyv3="$(python3 -V 2>&1)"
0054     export PYTHON_VERSION="${pyv3:7}"
0055     export PYSPARK_PYTHON="python3"
0056     export PYSPARK_DRIVER_PYTHON="python3"
0057 fi
0058
0059 # If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
0060 # It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
0061 if [ -n "${HADOOP_HOME}"  ] && [ -z "${SPARK_DIST_CLASSPATH}"  ]; then
0062   export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
0063 fi
0064
0065 if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
0066   SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
0067 fi
0068
0069 case "$1" in
0070   driver)
0071     shift 1
0072     CMD=(
0073       "$SPARK_HOME/bin/spark-submit"
0074       --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
0075       --deploy-mode client
0076       "$@"
0077     )
0078     ;;
0079   executor)
0080     shift 1
0081     CMD=(
0082       ${JAVA_HOME}/bin/java
0083       "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
0084       -Xms$SPARK_EXECUTOR_MEMORY
0085       -Xmx$SPARK_EXECUTOR_MEMORY
0086       -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
0087       org.apache.spark.executor.CoarseGrainedExecutorBackend
0088       --driver-url $SPARK_DRIVER_URL
0089       --executor-id $SPARK_EXECUTOR_ID
0090       --cores $SPARK_EXECUTOR_CORES
0091       --app-id $SPARK_APPLICATION_ID
0092       --hostname $SPARK_EXECUTOR_POD_IP
0093     )
0094     ;;
0095
0096   *)
0097     echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..."
0098     CMD=("$@")
0099     ;;
0100 esac
0101
0102 # Execute the container CMD under tini for better hygiene
0103 exec /usr/bin/tini -s -- "${CMD[@]}"