read/streaming/SparkDataStream.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.sql.connector.read.streaming;
0019
0020 import org.apache.spark.annotation.Evolving;
0021
0022 /**
0023  * The base interface representing a readable data stream in a Spark streaming query. It's
0024  * responsible to manage the offsets of the streaming source in the streaming query.
0025  *
0026  * Data sources should implement concrete data stream interfaces:
0027  * {@link MicroBatchStream} and {@link ContinuousStream}.
0028  *
0029  * @since 3.0.0
0030  */
0031 @Evolving
0032 public interface SparkDataStream {
0033
0034   /**
0035    * Returns the initial offset for a streaming query to start reading from. Note that the
0036    * streaming data source should not assume that it will start reading from its initial offset:
0037    * if Spark is restarting an existing query, it will restart from the check-pointed offset rather
0038    * than the initial one.
0039    */
0040   Offset initialOffset();
0041
0042   /**
0043    * Deserialize a JSON string into an Offset of the implementation-defined offset type.
0044    *
0045    * @throws IllegalArgumentException if the JSON does not encode a valid offset for this reader
0046    */
0047   Offset deserializeOffset(String json);
0048
0049   /**
0050    * Informs the source that Spark has completed processing all data for offsets less than or
0051    * equal to `end` and will only request offsets greater than `end` in the future.
0052    */
0053   void commit(Offset end);
0054
0055   /**
0056    * Stop this source and free any resources it has allocated.
0057    */
0058   void stop();
0059 }