api/plugin/DriverPlugin.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.api.plugin;
0019
0020 import java.util.Collections;
0021 import java.util.Map;
0022
0023 import org.apache.spark.SparkContext;
0024 import org.apache.spark.annotation.DeveloperApi;
0025
0026 /**
0027  * :: DeveloperApi ::
0028  * Driver component of a {@link SparkPlugin}.
0029  *
0030  * @since 3.0.0
0031  */
0032 @DeveloperApi
0033 public interface DriverPlugin {
0034
0035   /**
0036    * Initialize the plugin.
0037    * <p>
0038    * This method is called early in the initialization of the Spark driver. Explicitly, it is
0039    * called before the Spark driver's task scheduler is initialized. This means that a lot
0040    * of other Spark subsystems may yet not have been initialized. This call also blocks driver
0041    * initialization.
0042    * <p>
0043    * It's recommended that plugins be careful about what operations are performed in this call,
0044    * preferrably performing expensive operations in a separate thread, or postponing them until
0045    * the application has fully started.
0046    *
0047    * @param sc The SparkContext loading the plugin.
0048    * @param pluginContext Additional plugin-specific about the Spark application where the plugin
0049    *                      is running.
0050    * @return A map that will be provided to the {@link ExecutorPlugin#init(PluginContext,Map)}
0051    *         method.
0052    */
0053   default Map<String, String> init(SparkContext sc, PluginContext pluginContext) {
0054     return Collections.emptyMap();
0055   }
0056
0057   /**
0058    * Register metrics published by the plugin with Spark's metrics system.
0059    * <p>
0060    * This method is called later in the initialization of the Spark application, after most
0061    * subsystems are up and the application ID is known. If there are metrics registered in
0062    * the registry ({@link PluginContext#metricRegistry()}), then a metrics source with the
0063    * plugin name will be created.
0064    * <p>
0065    * Note that even though the metric registry is still accessible after this method is called,
0066    * registering new metrics after this method is called may result in the metrics not being
0067    * available.
0068    *
0069    * @param appId The application ID from the cluster manager.
0070    * @param pluginContext Additional plugin-specific about the Spark application where the plugin
0071    *                      is running.
0072    */
0073   default void registerMetrics(String appId, PluginContext pluginContext) {}
0074
0075   /**
0076    * RPC message handler.
0077    * <p>
0078    * Plugins can use Spark's RPC system to send messages from executors to the driver (but not
0079    * the other way around, currently). Messages sent by the executor component of the plugin will
0080    * be delivered to this method, and the returned value will be sent back to the executor as
0081    * the reply, if the executor has requested one.
0082    * <p>
0083    * Any exception thrown will be sent back to the executor as an error, in case it is expecting
0084    * a reply. In case a reply is not expected, a log message will be written to the driver log.
0085    * <p>
0086    * The implementation of this handler should be thread-safe.
0087    * <p>
0088    * Note all plugins share RPC dispatch threads, and this method is called synchronously. So
0089    * performing expensive operations in this handler may affect the operation of other active
0090    * plugins. Internal Spark endpoints are not directly affected, though, since they use different
0091    * threads.
0092    * <p>
0093    * Spark guarantees that the driver component will be ready to receive messages through this
0094    * handler when executors are started.
0095    *
0096    * @param message The incoming message.
0097    * @return Value to be returned to the caller. Ignored if the caller does not expect a reply.
0098    */
0099   default Object receive(Object message) throws Exception {
0100     throw new UnsupportedOperationException();
0101   }
0102
0103   /**
0104    * Informs the plugin that the Spark application is shutting down.
0105    * <p>
0106    * This method is called during the driver shutdown phase. It is recommended that plugins
0107    * not use any Spark functions (e.g. send RPC messages) during this call.
0108    */
0109   default void shutdown() {}
0110
0111 }