shuffle/api/ShuffleMapOutputWriter.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.shuffle.api;
0019
0020 import java.io.IOException;
0021
0022 import org.apache.spark.annotation.Private;
0023
0024 /**
0025  * :: Private ::
0026  * A top-level writer that returns child writers for persisting the output of a map task,
0027  * and then commits all of the writes as one atomic operation.
0028  *
0029  * @since 3.0.0
0030  */
0031 @Private
0032 public interface ShuffleMapOutputWriter {
0033
0034   /**
0035    * Creates a writer that can open an output stream to persist bytes targeted for a given reduce
0036    * partition id.
0037    * <p>
0038    * The chunk corresponds to bytes in the given reduce partition. This will not be called twice
0039    * for the same partition within any given map task. The partition identifier will be in the
0040    * range of precisely 0 (inclusive) to numPartitions (exclusive), where numPartitions was
0041    * provided upon the creation of this map output writer via
0042    * {@link ShuffleExecutorComponents#createMapOutputWriter(int, long, int)}.
0043    * <p>
0044    * Calls to this method will be invoked with monotonically increasing reducePartitionIds; each
0045    * call to this method will be called with a reducePartitionId that is strictly greater than
0046    * the reducePartitionIds given to any previous call to this method. This method is not
0047    * guaranteed to be called for every partition id in the above described range. In particular,
0048    * no guarantees are made as to whether or not this method will be called for empty partitions.
0049    */
0050   ShufflePartitionWriter getPartitionWriter(int reducePartitionId) throws IOException;
0051
0052   /**
0053    * Commits the writes done by all partition writers returned by all calls to this object's
0054    * {@link #getPartitionWriter(int)}, and returns the number of bytes written for each
0055    * partition.
0056    * <p>
0057    * This should ensure that the writes conducted by this module's partition writers are
0058    * available to downstream reduce tasks. If this method throws any exception, this module's
0059    * {@link #abort(Throwable)} method will be invoked before propagating the exception.
0060    * <p>
0061    * This can also close any resources and clean up temporary state if necessary.
0062    * <p>
0063    * The returned array should contain, for each partition from (0) to (numPartitions - 1), the
0064    * number of bytes written by the partition writer for that partition id.
0065    */
0066   long[] commitAllPartitions() throws IOException;
0067
0068   /**
0069    * Abort all of the writes done by any writers returned by {@link #getPartitionWriter(int)}.
0070    * <p>
0071    * This should invalidate the results of writing bytes. This can also close any resources and
0072    * clean up temporary state if necessary.
0073    */
0074   void abort(Throwable error) throws IOException;
0075 }