0001 <?xml version="1.0" encoding="UTF-8"?>
0002 <!--
0003 ~ Licensed to the Apache Software Foundation (ASF) under one or more
0004 ~ contributor license agreements. See the NOTICE file distributed with
0005 ~ this work for additional information regarding copyright ownership.
0006 ~ The ASF licenses this file to You under the Apache License, Version 2.0
0007 ~ (the "License"); you may not use this file except in compliance with
0008 ~ the License. You may obtain a copy of the License at
0009 ~
0010 ~ http://www.apache.org/licenses/LICENSE-2.0
0011 ~
0012 ~ Unless required by applicable law or agreed to in writing, software
0013 ~ distributed under the License is distributed on an "AS IS" BASIS,
0014 ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0015 ~ See the License for the specific language governing permissions and
0016 ~ limitations under the License.
0017 -->
0018 <project xmlns="http://maven.apache.org/POM/4.0.0"
0019 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
0020 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
0021 <modelVersion>4.0.0</modelVersion>
0022 <parent>
0023 <groupId>org.apache.spark</groupId>
0024 <artifactId>spark-parent_2.12</artifactId>
0025 <version>3.0.0</version>
0026 <relativePath>../pom.xml</relativePath>
0027 </parent>
0028
0029 <artifactId>spark-hadoop-cloud_2.12</artifactId>
0030 <packaging>jar</packaging>
0031 <name>Spark Project Hadoop Cloud Integration</name>
0032 <description>
0033 Contains Hadoop JARs and transitive dependencies needed to interact with cloud infrastructures.
0034 </description>
0035 <properties>
0036 <sbt.project.name>hadoop-cloud</sbt.project.name>
0037 </properties>
0038
0039 <build>
0040 <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
0041 <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
0042 </build>
0043
0044 <dependencies>
0045 <!--used during compilation but not exported as transitive dependencies-->
0046 <dependency>
0047 <groupId>org.apache.spark</groupId>
0048 <artifactId>spark-sql_${scala.binary.version}</artifactId>
0049 <version>${project.version}</version>
0050 <scope>provided</scope>
0051 </dependency>
0052 <dependency>
0053 <groupId>org.apache.spark</groupId>
0054 <artifactId>spark-core_${scala.binary.version}</artifactId>
0055 <version>${project.version}</version>
0056 <type>test-jar</type>
0057 <scope>test</scope>
0058 </dependency>
0059 <dependency>
0060 <groupId>org.apache.hadoop</groupId>
0061 <artifactId>hadoop-client</artifactId>
0062 <version>${hadoop.version}</version>
0063 <scope>provided</scope>
0064 </dependency>
0065 <!--
0066 the AWS module pulls in jackson; its transitive dependencies can create
0067 intra-jackson-module version problems.
0068 -->
0069 <dependency>
0070 <groupId>org.apache.hadoop</groupId>
0071 <artifactId>hadoop-aws</artifactId>
0072 <version>${hadoop.version}</version>
0073 <scope>${hadoop.deps.scope}</scope>
0074 <exclusions>
0075 <exclusion>
0076 <groupId>org.apache.hadoop</groupId>
0077 <artifactId>hadoop-common</artifactId>
0078 </exclusion>
0079 <exclusion>
0080 <groupId>commons-logging</groupId>
0081 <artifactId>commons-logging</artifactId>
0082 </exclusion>
0083 <exclusion>
0084 <groupId>org.codehaus.jackson</groupId>
0085 <artifactId>jackson-mapper-asl</artifactId>
0086 </exclusion>
0087 <exclusion>
0088 <groupId>org.codehaus.jackson</groupId>
0089 <artifactId>jackson-core-asl</artifactId>
0090 </exclusion>
0091 <exclusion>
0092 <groupId>com.fasterxml.jackson.core</groupId>
0093 <artifactId>jackson-core</artifactId>
0094 </exclusion>
0095 <exclusion>
0096 <groupId>com.fasterxml.jackson.core</groupId>
0097 <artifactId>jackson-databind</artifactId>
0098 </exclusion>
0099 <exclusion>
0100 <groupId>com.fasterxml.jackson.core</groupId>
0101 <artifactId>jackson-annotations</artifactId>
0102 </exclusion>
0103 <!-- Keep old SDK out of the assembly to avoid conflict with Kinesis module -->
0104 <exclusion>
0105 <groupId>com.amazonaws</groupId>
0106 <artifactId>aws-java-sdk</artifactId>
0107 </exclusion>
0108 </exclusions>
0109 </dependency>
0110 <dependency>
0111 <groupId>org.apache.hadoop</groupId>
0112 <artifactId>hadoop-openstack</artifactId>
0113 <version>${hadoop.version}</version>
0114 <scope>${hadoop.deps.scope}</scope>
0115 <exclusions>
0116 <exclusion>
0117 <groupId>org.apache.hadoop</groupId>
0118 <artifactId>hadoop-common</artifactId>
0119 </exclusion>
0120 <exclusion>
0121 <groupId>commons-logging</groupId>
0122 <artifactId>commons-logging</artifactId>
0123 </exclusion>
0124 <exclusion>
0125 <groupId>junit</groupId>
0126 <artifactId>junit</artifactId>
0127 </exclusion>
0128 <exclusion>
0129 <groupId>org.mockito</groupId>
0130 <artifactId>mockito-all</artifactId>
0131 </exclusion>
0132 </exclusions>
0133 </dependency>
0134
0135 <!--
0136 Add joda time to ensure that anything downstream which doesn't pull in spark-hive
0137 gets the correct joda time artifact, so doesn't have auth failures on later Java 8 JVMs
0138 -->
0139 <dependency>
0140 <groupId>joda-time</groupId>
0141 <artifactId>joda-time</artifactId>
0142 <scope>${hadoop.deps.scope}</scope>
0143 </dependency>
0144 <!-- explicitly declare the jackson artifacts desired -->
0145 <dependency>
0146 <groupId>com.fasterxml.jackson.core</groupId>
0147 <artifactId>jackson-databind</artifactId>
0148 <scope>${hadoop.deps.scope}</scope>
0149 </dependency>
0150 <dependency>
0151 <groupId>com.fasterxml.jackson.core</groupId>
0152 <artifactId>jackson-annotations</artifactId>
0153 <scope>${hadoop.deps.scope}</scope>
0154 </dependency>
0155 <dependency>
0156 <groupId>com.fasterxml.jackson.dataformat</groupId>
0157 <artifactId>jackson-dataformat-cbor</artifactId>
0158 <version>${fasterxml.jackson.version}</version>
0159 </dependency>
0160 <!--Explicit declaration to force in Spark version into transitive dependencies -->
0161 <dependency>
0162 <groupId>org.apache.httpcomponents</groupId>
0163 <artifactId>httpclient</artifactId>
0164 <scope>${hadoop.deps.scope}</scope>
0165 </dependency>
0166 <!--Explicit declaration to force in Spark version into transitive dependencies -->
0167 <dependency>
0168 <groupId>org.apache.httpcomponents</groupId>
0169 <artifactId>httpcore</artifactId>
0170 <scope>${hadoop.deps.scope}</scope>
0171 </dependency>
0172 <dependency>
0173 <groupId>org.apache.hadoop</groupId>
0174 <artifactId>hadoop-azure</artifactId>
0175 <version>${hadoop.version}</version>
0176 <scope>${hadoop.deps.scope}</scope>
0177 <exclusions>
0178 <exclusion>
0179 <groupId>org.apache.hadoop</groupId>
0180 <artifactId>hadoop-common</artifactId>
0181 </exclusion>
0182 <exclusion>
0183 <groupId>org.codehaus.jackson</groupId>
0184 <artifactId>jackson-mapper-asl</artifactId>
0185 </exclusion>
0186 <exclusion>
0187 <groupId>com.fasterxml.jackson.core</groupId>
0188 <artifactId>jackson-core</artifactId>
0189 </exclusion>
0190 <exclusion>
0191 <groupId>com.google.guava</groupId>
0192 <artifactId>guava</artifactId>
0193 </exclusion>
0194 </exclusions>
0195 </dependency>
0196 </dependencies>
0197
0198 <profiles>
0199
0200 <!--
0201 Hadoop 3 simplifies the classpath, and adds a new committer base class which
0202 enables store-specific committers.
0203 -->
0204 <profile>
0205 <id>hadoop-3.2</id>
0206 <properties>
0207 <extra.source.dir>src/hadoop-3/main/scala</extra.source.dir>
0208 <extra.testsource.dir>src/hadoop-3/test/scala</extra.testsource.dir>
0209 </properties>
0210
0211 <build>
0212 <plugins>
0213 <plugin>
0214 <groupId>org.codehaus.mojo</groupId>
0215 <artifactId>build-helper-maven-plugin</artifactId>
0216 <executions>
0217 <execution>
0218 <id>add-scala-sources</id>
0219 <phase>generate-sources</phase>
0220 <goals>
0221 <goal>add-source</goal>
0222 </goals>
0223 <configuration>
0224 <sources>
0225 <source>${extra.source.dir}</source>
0226 </sources>
0227 </configuration>
0228 </execution>
0229 <execution>
0230 <id>add-scala-test-sources</id>
0231 <phase>generate-test-sources</phase>
0232 <goals>
0233 <goal>add-test-source</goal>
0234 </goals>
0235 <configuration>
0236 <sources>
0237 <source>${extra.testsource.dir}</source>
0238 </sources>
0239 </configuration>
0240 </execution>
0241 </executions>
0242 </plugin>
0243 </plugins>
0244 </build>
0245 <dependencies>
0246 <!--
0247 There's now a hadoop-cloud-storage which transitively pulls in the store JARs,
0248 but it still needs some selective exclusion across versions, especially 3.0.x.
0249 -->
0250 <dependency>
0251 <groupId>org.apache.hadoop</groupId>
0252 <artifactId>hadoop-cloud-storage</artifactId>
0253 <version>${hadoop.version}</version>
0254 <scope>${hadoop.deps.scope}</scope>
0255 <exclusions>
0256 <exclusion>
0257 <groupId>org.apache.hadoop</groupId>
0258 <artifactId>hadoop-common</artifactId>
0259 </exclusion>
0260 <exclusion>
0261 <groupId>org.codehaus.jackson</groupId>
0262 <artifactId>jackson-mapper-asl</artifactId>
0263 </exclusion>
0264 <exclusion>
0265 <groupId>com.fasterxml.jackson.core</groupId>
0266 <artifactId>jackson-core</artifactId>
0267 </exclusion>
0268 <exclusion>
0269 <groupId>com.google.guava</groupId>
0270 <artifactId>guava</artifactId>
0271 </exclusion>
0272 </exclusions>
0273 </dependency>
0274 <!--
0275 The jetty declarations are made
0276 (a) to keep that jetty-util-ajax version in sync with the rest of Spark.
0277 (b) to minimise the effects which Spark's jetty shading has on the
0278 availability of the jetty JARs on for hadoop-azure, which depends
0279 on them.
0280 -->
0281 <dependency>
0282 <groupId>org.eclipse.jetty</groupId>
0283 <artifactId>jetty-util</artifactId>
0284 <scope>${hadoop.deps.scope}</scope>
0285 </dependency>
0286 <dependency>
0287 <groupId>org.eclipse.jetty</groupId>
0288 <artifactId>jetty-util-ajax</artifactId>
0289 <version>${jetty.version}</version>
0290 <scope>${hadoop.deps.scope}</scope>
0291 </dependency>
0292 </dependencies>
0293 </profile>
0294
0295 </profiles>
0296
0297 </project>
