0001 #!/usr/bin/env bash
0002
0003 #
0004 # Licensed to the Apache Software Foundation (ASF) under one or more
0005 # contributor license agreements. See the NOTICE file distributed with
0006 # this work for additional information regarding copyright ownership.
0007 # The ASF licenses this file to You under the Apache License, Version 2.0
0008 # (the "License"); you may not use this file except in compliance with
0009 # the License. You may obtain a copy of the License at
0010 #
0011 # http://www.apache.org/licenses/LICENSE-2.0
0012 #
0013 # Unless required by applicable law or agreed to in writing, software
0014 # distributed under the License is distributed on an "AS IS" BASIS,
0015 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0016 # See the License for the specific language governing permissions and
0017 # limitations under the License.
0018 #
0019
0020 # Determine the current working directory
0021 _DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
0022 # Preserve the calling directory
0023 _CALLING_DIR="$(pwd)"
0024 # Options used during compilation
0025 _COMPILE_JVM_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g"
0026
0027 # Installs any application tarball given a URL, the expected tarball name,
0028 # and, optionally, a checkable binary path to determine if the binary has
0029 # already been installed
0030 ## Arg1 - URL
0031 ## Arg2 - Tarball Name
0032 ## Arg3 - Checkable Binary
0033 install_app() {
0034 local remote_tarball="$1/$2"
0035 local local_tarball="${_DIR}/$2"
0036 local binary="${_DIR}/$3"
0037
0038 # setup `curl` and `wget` silent options if we're running on Jenkins
0039 local curl_opts="-L"
0040 local wget_opts=""
0041 if [ -n "$AMPLAB_JENKINS" ]; then
0042 curl_opts="-s ${curl_opts}"
0043 wget_opts="--quiet ${wget_opts}"
0044 else
0045 curl_opts="--progress-bar ${curl_opts}"
0046 wget_opts="--progress=bar:force ${wget_opts}"
0047 fi
0048
0049 if [ -z "$3" -o ! -f "$binary" ]; then
0050 # check if we already have the tarball
0051 # check if we have curl installed
0052 # download application
0053 [ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \
0054 echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2 && \
0055 curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
0056 # if the file still doesn't exist, lets try `wget` and cross our fingers
0057 [ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \
0058 echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2 && \
0059 wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
0060 # if both were unsuccessful, exit
0061 [ ! -f "${local_tarball}" ] && \
0062 echo -n "ERROR: Cannot download $2 with cURL or wget; " && \
0063 echo "please install manually and try again." && \
0064 exit 2
0065 cd "${_DIR}" && tar -xzf "$2"
0066 rm -rf "$local_tarball"
0067 fi
0068 }
0069
0070 # See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
0071 function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
0072
0073 # Determine the Maven version from the root pom.xml file and
0074 # install maven under the build/ folder if needed.
0075 install_mvn() {
0076 local MVN_VERSION=`grep "<maven.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
0077 MVN_BIN="$(command -v mvn)"
0078 if [ "$MVN_BIN" ]; then
0079 local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
0080 fi
0081 if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
0082 local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}
0083
0084 if [ $(command -v curl) ]; then
0085 local TEST_MIRROR_URL="${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz"
0086 if ! curl -L --output /dev/null --silent --head --fail "$TEST_MIRROR_URL" ; then
0087 # Fall back to archive.apache.org for older Maven
0088 echo "Falling back to archive.apache.org to download Maven"
0089 APACHE_MIRROR="https://archive.apache.org/dist"
0090 fi
0091 fi
0092
0093 install_app \
0094 "${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \
0095 "apache-maven-${MVN_VERSION}-bin.tar.gz" \
0096 "apache-maven-${MVN_VERSION}/bin/mvn"
0097
0098 MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
0099 fi
0100 }
0101
0102 # Install zinc under the build/ folder
0103 install_zinc() {
0104 local ZINC_VERSION=0.3.15
0105 ZINC_BIN="$(command -v zinc)"
0106 if [ "$ZINC_BIN" ]; then
0107 local ZINC_DETECTED_VERSION="$(zinc -version | head -n1 | awk '{print $5}')"
0108 fi
0109
0110 if [ $(version $ZINC_DETECTED_VERSION) -lt $(version $ZINC_VERSION) ]; then
0111 local zinc_path="zinc-${ZINC_VERSION}/bin/zinc"
0112 [ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
0113 local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
0114
0115 install_app \
0116 "${TYPESAFE_MIRROR}/zinc/${ZINC_VERSION}" \
0117 "zinc-${ZINC_VERSION}.tgz" \
0118 "${zinc_path}"
0119 ZINC_BIN="${_DIR}/${zinc_path}"
0120 fi
0121 }
0122
0123 # Determine the Scala version from the root pom.xml file, set the Scala URL,
0124 # and, with that, download the specific version of Scala necessary under
0125 # the build/ folder
0126 install_scala() {
0127 # determine the Scala version used in Spark
0128 local scala_binary_version=`grep "scala.binary.version" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
0129 local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | grep ${scala_binary_version} | head -n1 | awk -F '[<>]' '{print $3}'`
0130 local scala_bin="${_DIR}/scala-${scala_version}/bin/scala"
0131 local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
0132
0133 install_app \
0134 "${TYPESAFE_MIRROR}/scala/${scala_version}" \
0135 "scala-${scala_version}.tgz" \
0136 "scala-${scala_version}/bin/scala"
0137
0138 SCALA_COMPILER="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-compiler.jar"
0139 SCALA_LIBRARY="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-library.jar"
0140 }
0141
0142 # Setup healthy defaults for the Zinc port if none were provided from
0143 # the environment
0144 ZINC_PORT=${ZINC_PORT:-"3030"}
0145
0146 # Install the proper version of Scala, Zinc and Maven for the build
0147 install_zinc
0148 install_scala
0149 install_mvn
0150
0151 # Reset the current working directory
0152 cd "${_CALLING_DIR}"
0153
0154 # Now that zinc is ensured to be installed, check its status and, if its
0155 # not running or just installed, start it
0156 if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port ${ZINC_PORT}`" ]; then
0157 export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
0158 "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
0159 "${ZINC_BIN}" -start -port ${ZINC_PORT} \
0160 -server 127.0.0.1 -idle-timeout 3h \
0161 -scala-compiler "${SCALA_COMPILER}" \
0162 -scala-library "${SCALA_LIBRARY}" &>/dev/null
0163 fi
0164
0165 # Set any `mvn` options if not already present
0166 export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}
0167
0168 echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
0169
0170 # call the `mvn` command as usual
0171 # SPARK-25854
0172 "${MVN_BIN}" -DzincPort=${ZINC_PORT} "$@"
0173 MVN_RETCODE=$?
0174
0175 # Try to shut down zinc explicitly if the server is still running.
0176 "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
0177
0178 exit $MVN_RETCODE
