shuffle/protocol/ExecutorShuffleInfo.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.network.shuffle.protocol;
0019
0020 import java.util.Arrays;
0021 import java.util.Objects;
0022
0023 import com.fasterxml.jackson.annotation.JsonCreator;
0024 import com.fasterxml.jackson.annotation.JsonProperty;
0025 import io.netty.buffer.ByteBuf;
0026 import org.apache.commons.lang3.builder.ToStringBuilder;
0027 import org.apache.commons.lang3.builder.ToStringStyle;
0028
0029 import org.apache.spark.network.protocol.Encodable;
0030 import org.apache.spark.network.protocol.Encoders;
0031
0032 /** Contains all configuration necessary for locating the shuffle files of an executor. */
0033 public class ExecutorShuffleInfo implements Encodable {
0034   /** The base set of local directories that the executor stores its shuffle files in. */
0035   public final String[] localDirs;
0036   /** Number of subdirectories created within each localDir. */
0037   public final int subDirsPerLocalDir;
0038   /** Shuffle manager (SortShuffleManager) that the executor is using. */
0039   public final String shuffleManager;
0040
0041   @JsonCreator
0042   public ExecutorShuffleInfo(
0043       @JsonProperty("localDirs") String[] localDirs,
0044       @JsonProperty("subDirsPerLocalDir") int subDirsPerLocalDir,
0045       @JsonProperty("shuffleManager") String shuffleManager) {
0046     this.localDirs = localDirs;
0047     this.subDirsPerLocalDir = subDirsPerLocalDir;
0048     this.shuffleManager = shuffleManager;
0049   }
0050
0051   @Override
0052   public int hashCode() {
0053     return Objects.hash(subDirsPerLocalDir, shuffleManager) * 41 + Arrays.hashCode(localDirs);
0054   }
0055
0056   @Override
0057   public String toString() {
0058     return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
0059       .append("localDirs", Arrays.toString(localDirs))
0060       .append("subDirsPerLocalDir", subDirsPerLocalDir)
0061       .append("shuffleManager", shuffleManager)
0062       .toString();
0063   }
0064
0065   @Override
0066   public boolean equals(Object other) {
0067     if (other != null && other instanceof ExecutorShuffleInfo) {
0068       ExecutorShuffleInfo o = (ExecutorShuffleInfo) other;
0069       return Arrays.equals(localDirs, o.localDirs)
0070         && subDirsPerLocalDir == o.subDirsPerLocalDir
0071         && Objects.equals(shuffleManager, o.shuffleManager);
0072     }
0073     return false;
0074   }
0075
0076   @Override
0077   public int encodedLength() {
0078     return Encoders.StringArrays.encodedLength(localDirs)
0079         + 4 // int
0080         + Encoders.Strings.encodedLength(shuffleManager);
0081   }
0082
0083   @Override
0084   public void encode(ByteBuf buf) {
0085     Encoders.StringArrays.encode(buf, localDirs);
0086     buf.writeInt(subDirsPerLocalDir);
0087     Encoders.Strings.encode(buf, shuffleManager);
0088   }
0089
0090   public static ExecutorShuffleInfo decode(ByteBuf buf) {
0091     String[] localDirs = Encoders.StringArrays.decode(buf);
0092     int subDirsPerLocalDir = buf.readInt();
0093     String shuffleManager = Encoders.Strings.decode(buf);
0094     return new ExecutorShuffleInfo(localDirs, subDirsPerLocalDir, shuffleManager);
0095   }
0096 }