|
||||
0001 /* 0002 * Licensed to the Apache Software Foundation (ASF) under one or more 0003 * contributor license agreements. See the NOTICE file distributed with 0004 * this work for additional information regarding copyright ownership. 0005 * The ASF licenses this file to You under the Apache License, Version 2.0 0006 * (the "License"); you may not use this file except in compliance with 0007 * the License. You may obtain a copy of the License at 0008 * 0009 * http://www.apache.org/licenses/LICENSE-2.0 0010 * 0011 * Unless required by applicable law or agreed to in writing, software 0012 * distributed under the License is distributed on an "AS IS" BASIS, 0013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 0014 * See the License for the specific language governing permissions and 0015 * limitations under the License. 0016 */ 0017 0018 package org.apache.spark.sql.connector.read; 0019 0020 import java.io.Serializable; 0021 0022 import org.apache.spark.annotation.Evolving; 0023 import org.apache.spark.sql.catalyst.InternalRow; 0024 import org.apache.spark.sql.vectorized.ColumnarBatch; 0025 0026 /** 0027 * A factory used to create {@link PartitionReader} instances. 0028 * 0029 * If Spark fails to execute any methods in the implementations of this interface or in the returned 0030 * {@link PartitionReader} (by throwing an exception), corresponding Spark task would fail and 0031 * get retried until hitting the maximum retry times. 0032 * 0033 * @since 3.0.0 0034 */ 0035 @Evolving 0036 public interface PartitionReaderFactory extends Serializable { 0037 0038 /** 0039 * Returns a row-based partition reader to read data from the given {@link InputPartition}. 0040 * 0041 * Implementations probably need to cast the input partition to the concrete 0042 * {@link InputPartition} class defined for the data source. 0043 */ 0044 PartitionReader<InternalRow> createReader(InputPartition partition); 0045 0046 /** 0047 * Returns a columnar partition reader to read data from the given {@link InputPartition}. 0048 * 0049 * Implementations probably need to cast the input partition to the concrete 0050 * {@link InputPartition} class defined for the data source. 0051 */ 0052 default PartitionReader<ColumnarBatch> createColumnarReader(InputPartition partition) { 0053 throw new UnsupportedOperationException("Cannot create columnar reader."); 0054 } 0055 0056 /** 0057 * Returns true if the given {@link InputPartition} should be read by Spark in a columnar way. 0058 * This means, implementations must also implement {@link #createColumnarReader(InputPartition)} 0059 * for the input partitions that this method returns true. 0060 * 0061 * As of Spark 2.4, Spark can only read all input partition in a columnar way, or none of them. 0062 * Data source can't mix columnar and row-based partitions. This may be relaxed in future 0063 * versions. 0064 */ 0065 default boolean supportColumnarReads(InputPartition partition) { 0066 return false; 0067 } 0068 }
[ Source navigation ] | [ Diff markup ] | [ Identifier search ] | [ general search ] |
This page was automatically generated by the 2.1.0 LXR engine. The LXR team |