streaming/util/WriteAheadLog.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.streaming.util;
0019
0020 import java.nio.ByteBuffer;
0021 import java.util.Iterator;
0022
0023 /**
0024  * :: DeveloperApi ::
0025  *
0026  * This abstract class represents a write ahead log (aka journal) that is used by Spark Streaming
0027  * to save the received data (by receivers) and associated metadata to a reliable storage, so that
0028  * they can be recovered after driver failures. See the Spark documentation for more information
0029  * on how to plug in your own custom implementation of a write ahead log.
0030  */
0031 @org.apache.spark.annotation.DeveloperApi
0032 public abstract class WriteAheadLog {
0033   /**
0034    * Write the record to the log and return a record handle, which contains all the information
0035    * necessary to read back the written record. The time is used to the index the record,
0036    * such that it can be cleaned later. Note that implementations of this abstract class must
0037    * ensure that the written data is durable and readable (using the record handle) by the
0038    * time this function returns.
0039    */
0040   public abstract WriteAheadLogRecordHandle write(ByteBuffer record, long time);
0041
0042   /**
0043    * Read a written record based on the given record handle.
0044    */
0045   public abstract ByteBuffer read(WriteAheadLogRecordHandle handle);
0046
0047   /**
0048    * Read and return an iterator of all the records that have been written but not yet cleaned up.
0049    */
0050   public abstract Iterator<ByteBuffer> readAll();
0051
0052   /**
0053    * Clean all the records that are older than the threshold time. It can wait for
0054    * the completion of the deletion.
0055    */
0056   public abstract void clean(long threshTime, boolean waitForCompletion);
0057
0058   /**
0059    * Close this log and release any resources. It must be idempotent.
0060    */
0061   public abstract void close();
0062 }