integration-tests/dev/dev-run-integration-tests.sh

0001 #!/usr/bin/env bash
0002
0003 #
0004 # Licensed to the Apache Software Foundation (ASF) under one or more
0005 # contributor license agreements.  See the NOTICE file distributed with
0006 # this work for additional information regarding copyright ownership.
0007 # The ASF licenses this file to You under the Apache License, Version 2.0
0008 # (the "License"); you may not use this file except in compliance with
0009 # the License.  You may obtain a copy of the License at
0010 #
0011 #    http://www.apache.org/licenses/LICENSE-2.0
0012 #
0013 # Unless required by applicable law or agreed to in writing, software
0014 # distributed under the License is distributed on an "AS IS" BASIS,
0015 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0016 # See the License for the specific language governing permissions and
0017 # limitations under the License.
0018 #
0019 set -xo errexit
0020 TEST_ROOT_DIR=$(git rev-parse --show-toplevel)
0021
0022 DEPLOY_MODE="minikube"
0023 IMAGE_REPO="docker.io/kubespark"
0024 SPARK_TGZ="N/A"
0025 IMAGE_TAG="N/A"
0026 JAVA_IMAGE_TAG=
0027 BASE_IMAGE_NAME=
0028 JVM_IMAGE_NAME=
0029 PYTHON_IMAGE_NAME=
0030 R_IMAGE_NAME=
0031 SPARK_MASTER=
0032 NAMESPACE=
0033 SERVICE_ACCOUNT=
0034 CONTEXT=
0035 INCLUDE_TAGS="k8s"
0036 EXCLUDE_TAGS=
0037 JAVA_VERSION="8"
0038 HADOOP_PROFILE="hadoop-2.7"
0039 MVN="$TEST_ROOT_DIR/build/mvn"
0040
0041 SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version 2>/dev/null\
0042     | grep -v "INFO"\
0043     | grep -v "WARNING"\
0044     | tail -n 1)
0045
0046 # Parse arguments
0047 while (( "$#" )); do
0048   case $1 in
0049     --image-repo)
0050       IMAGE_REPO="$2"
0051       shift
0052       ;;
0053     --image-tag)
0054       IMAGE_TAG="$2"
0055       shift
0056       ;;
0057     --java-image-tag)
0058       JAVA_IMAGE_TAG="$2"
0059       shift
0060       ;;
0061     --deploy-mode)
0062       DEPLOY_MODE="$2"
0063       shift
0064       ;;
0065     --spark-tgz)
0066       SPARK_TGZ="$2"
0067       shift
0068       ;;
0069     --spark-master)
0070       SPARK_MASTER="$2"
0071       shift
0072       ;;
0073     --namespace)
0074       NAMESPACE="$2"
0075       shift
0076       ;;
0077     --service-account)
0078       SERVICE_ACCOUNT="$2"
0079       shift
0080       ;;
0081     --context)
0082       CONTEXT="$2"
0083       shift
0084       ;;
0085     --include-tags)
0086       INCLUDE_TAGS="k8s,$2"
0087       shift
0088       ;;
0089     --exclude-tags)
0090       EXCLUDE_TAGS="$2"
0091       shift
0092       ;;
0093     --base-image-name)
0094       BASE_IMAGE_NAME="$2"
0095       shift
0096       ;;
0097     --jvm-image-name)
0098       JVM_IMAGE_NAME="$2"
0099       shift
0100       ;;
0101     --python-image-name)
0102       PYTHON_IMAGE_NAME="$2"
0103       shift
0104       ;;
0105     --r-image-name)
0106       R_IMAGE_NAME="$2"
0107       shift
0108       ;;
0109     --java-version)
0110       JAVA_VERSION="$2"
0111       shift
0112       ;;
0113     --hadoop-profile)
0114       HADOOP_PROFILE="$2"
0115       shift
0116       ;;
0117     *)
0118       break
0119       ;;
0120   esac
0121   shift
0122 done
0123
0124 properties=(
0125   -Djava.version=$JAVA_VERSION \
0126   -Dspark.kubernetes.test.sparkTgz=$SPARK_TGZ \
0127   -Dspark.kubernetes.test.imageTag=$IMAGE_TAG \
0128   -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \
0129   -Dspark.kubernetes.test.deployMode=$DEPLOY_MODE \
0130   -Dtest.include.tags=$INCLUDE_TAGS
0131 )
0132
0133 if [ -n "$JAVA_IMAGE_TAG" ];
0134 then
0135   properties=( ${properties[@]} -Dspark.kubernetes.test.javaImageTag=$JAVA_IMAGE_TAG )
0136 fi
0137
0138 if [ -n "$NAMESPACE" ];
0139 then
0140   properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE )
0141 fi
0142
0143 if [ -n "$SERVICE_ACCOUNT" ];
0144 then
0145   properties=( ${properties[@]} -Dspark.kubernetes.test.serviceAccountName=$SERVICE_ACCOUNT )
0146 fi
0147
0148 if [ -n "$CONTEXT" ];
0149 then
0150   properties=( ${properties[@]} -Dspark.kubernetes.test.kubeConfigContext=$CONTEXT )
0151 fi
0152
0153 if [ -n "$SPARK_MASTER" ];
0154 then
0155   properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER )
0156 fi
0157
0158 if [ -n "$EXCLUDE_TAGS" ];
0159 then
0160   properties=( ${properties[@]} -Dtest.exclude.tags=$EXCLUDE_TAGS )
0161 fi
0162
0163 BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-spark}
0164 JVM_IMAGE_NAME=${JVM_IMAGE_NAME:-${BASE_IMAGE_NAME}}
0165 PYTHON_IMAGE_NAME=${PYTHON_IMAGE_NAME:-${BASE_IMAGE_NAME}-py}
0166 R_IMAGE_NAME=${R_IMAGE_NAME:-${BASE_IMAGE_NAME}-r}
0167
0168 properties+=(
0169   -Dspark.kubernetes.test.jvmImage=$JVM_IMAGE_NAME
0170   -Dspark.kubernetes.test.pythonImage=$PYTHON_IMAGE_NAME
0171   -Dspark.kubernetes.test.rImage=$R_IMAGE_NAME
0172 )
0173
0174 $TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pscala-$SCALA_VERSION -P$HADOOP_PROFILE -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}