0001 <?xml version="1.0" encoding="UTF-8"?>
0002 <!--
0003 ~ Licensed to the Apache Software Foundation (ASF) under one or more
0004 ~ contributor license agreements. See the NOTICE file distributed with
0005 ~ this work for additional information regarding copyright ownership.
0006 ~ The ASF licenses this file to You under the Apache License, Version 2.0
0007 ~ (the "License"); you may not use this file except in compliance with
0008 ~ the License. You may obtain a copy of the License at
0009 ~
0010 ~ http://www.apache.org/licenses/LICENSE-2.0
0011 ~
0012 ~ Unless required by applicable law or agreed to in writing, software
0013 ~ distributed under the License is distributed on an "AS IS" BASIS,
0014 ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0015 ~ See the License for the specific language governing permissions and
0016 ~ limitations under the License.
0017 -->
0018
0019 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
0020 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
0021 <modelVersion>4.0.0</modelVersion>
0022 <parent>
0023 <groupId>org.apache.spark</groupId>
0024 <artifactId>spark-parent_2.12</artifactId>
0025 <version>3.0.0</version>
0026 <relativePath>../../pom.xml</relativePath>
0027 </parent>
0028
0029 <artifactId>spark-network-yarn_2.12</artifactId>
0030 <packaging>jar</packaging>
0031 <name>Spark Project YARN Shuffle Service</name>
0032 <url>http://spark.apache.org/</url>
0033 <properties>
0034 <sbt.project.name>network-yarn</sbt.project.name>
0035 <!-- Make sure all Hadoop dependencies are provided to avoid repackaging. -->
0036 <hadoop.deps.scope>provided</hadoop.deps.scope>
0037 <shuffle.jar>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</shuffle.jar>
0038 <shade>org/sparkproject/</shade>
0039 </properties>
0040
0041 <dependencies>
0042 <!-- Core dependencies -->
0043 <dependency>
0044 <groupId>org.apache.spark</groupId>
0045 <artifactId>spark-network-shuffle_${scala.binary.version}</artifactId>
0046 <version>${project.version}</version>
0047 </dependency>
0048 <dependency>
0049 <groupId>org.apache.spark</groupId>
0050 <artifactId>spark-tags_${scala.binary.version}</artifactId>
0051 <scope>test</scope>
0052 </dependency>
0053
0054 <!--
0055 This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
0056 them will yield errors.
0057 -->
0058 <dependency>
0059 <groupId>org.apache.spark</groupId>
0060 <artifactId>spark-tags_${scala.binary.version}</artifactId>
0061 <type>test-jar</type>
0062 <scope>test</scope>
0063 </dependency>
0064
0065 <!-- Provided dependencies -->
0066 <dependency>
0067 <groupId>org.apache.hadoop</groupId>
0068 <artifactId>hadoop-client</artifactId>
0069 </dependency>
0070 <dependency>
0071 <groupId>org.slf4j</groupId>
0072 <artifactId>slf4j-api</artifactId>
0073 <scope>provided</scope>
0074 </dependency>
0075 </dependencies>
0076
0077 <build>
0078 <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
0079 <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
0080 <plugins>
0081 <plugin>
0082 <groupId>org.apache.maven.plugins</groupId>
0083 <artifactId>maven-shade-plugin</artifactId>
0084 <configuration>
0085 <shadedArtifactAttached>false</shadedArtifactAttached>
0086 <outputFile>${shuffle.jar}</outputFile>
0087 <artifactSet>
0088 <includes>
0089 <include>*:*</include>
0090 </includes>
0091 <excludes>
0092 <exclude>org.scala-lang:scala-library</exclude>
0093 </excludes>
0094 </artifactSet>
0095 <filters>
0096 <filter>
0097 <artifact>*:*</artifact>
0098 <excludes>
0099 <exclude>META-INF/*.SF</exclude>
0100 <exclude>META-INF/*.DSA</exclude>
0101 <exclude>META-INF/*.RSA</exclude>
0102 </excludes>
0103 </filter>
0104 </filters>
0105 <relocations combine.children="append">
0106 <relocation>
0107 <pattern>com.fasterxml.jackson</pattern>
0108 <shadedPattern>${spark.shade.packageName}.com.fasterxml.jackson</shadedPattern>
0109 <includes>
0110 <include>com.fasterxml.jackson.**</include>
0111 </includes>
0112 </relocation>
0113 <relocation>
0114 <pattern>io.netty</pattern>
0115 <shadedPattern>${spark.shade.packageName}.io.netty</shadedPattern>
0116 <includes>
0117 <include>io.netty.**</include>
0118 </includes>
0119 </relocation>
0120 </relocations>
0121 </configuration>
0122 <executions>
0123 <execution>
0124 <phase>package</phase>
0125 <goals>
0126 <goal>shade</goal>
0127 </goals>
0128 </execution>
0129 </executions>
0130 </plugin>
0131 <!-- shade the native netty libs as well -->
0132 <plugin>
0133 <groupId>org.codehaus.mojo</groupId>
0134 <artifactId>build-helper-maven-plugin</artifactId>
0135 <executions>
0136 <execution>
0137 <id>regex-property</id>
0138 <goals>
0139 <goal>regex-property</goal>
0140 </goals>
0141 <configuration>
0142 <name>spark.shade.native.packageName</name>
0143 <value>${spark.shade.packageName}</value>
0144 <regex>\.</regex>
0145 <replacement>_</replacement>
0146 <failIfNoMatch>true</failIfNoMatch>
0147 </configuration>
0148 </execution>
0149 </executions>
0150 </plugin>
0151 <plugin>
0152 <groupId>org.apache.maven.plugins</groupId>
0153 <artifactId>maven-antrun-plugin</artifactId>
0154 <executions>
0155 <execution>
0156 <id>unpack</id>
0157 <phase>package</phase>
0158 <configuration>
0159 <target>
0160 <echo message="Shade netty native libraries to ${spark.shade.native.packageName}" />
0161 <unzip src="${shuffle.jar}" dest="${project.build.directory}/exploded/" />
0162 <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_epoll_x86_64.so"
0163 tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_x86_64.so" />
0164 <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib"
0165 tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib" />
0166 <jar destfile="${shuffle.jar}" basedir="${project.build.directory}/exploded" />
0167 </target>
0168 </configuration>
0169 <goals>
0170 <goal>run</goal>
0171 </goals>
0172 </execution>
0173 <!-- probes to validate that those dependencies which must be shaded are -->
0174 <execution>
0175 <phase>verify</phase>
0176 <goals>
0177 <goal>run</goal>
0178 </goals>
0179 <configuration>
0180 <target>
0181 <macrodef name="shaded">
0182 <attribute name="resource"/>
0183 <sequential>
0184 <fail message="Not found ${shade}@{resource}">
0185 <condition>
0186 <not>
0187 <resourceexists>
0188 <zipentry zipfile="${shuffle.jar}" name="${shade}@{resource}"/>
0189 </resourceexists>
0190 </not>
0191 </condition>
0192 </fail>
0193 </sequential>
0194 </macrodef>
0195 <echo>Verifying dependency shading</echo>
0196 <shaded resource="com/fasterxml/jackson/core/JsonParser.class" />
0197 <shaded resource="com/fasterxml/jackson/annotation/JacksonAnnotation.class" />
0198 <shaded resource="com/fasterxml/jackson/databind/JsonSerializer.class" />
0199 </target>
0200 </configuration>
0201 </execution>
0202 </executions>
0203 </plugin>
0204 </plugins>
0205 </build>
0206 </project>
