0001 #
0002 # Licensed to the Apache Software Foundation (ASF) under one or more
0003 # contributor license agreements. See the NOTICE file distributed with
0004 # this work for additional information regarding copyright ownership.
0005 # The ASF licenses this file to You under the Apache License, Version 2.0
0006 # (the "License"); you may not use this file except in compliance with
0007 # the License. You may obtain a copy of the License at
0008 #
0009 # http://www.apache.org/licenses/LICENSE-2.0
0010 #
0011 # Unless required by applicable law or agreed to in writing, software
0012 # distributed under the License is distributed on an "AS IS" BASIS,
0013 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014 # See the License for the specific language governing permissions and
0015 # limitations under the License.
0016 #
0017
0018 # syntax: [instance].sink|source.[name].[options]=[value]
0019
0020 # This file configures Spark's internal metrics system. The metrics system is
0021 # divided into instances which correspond to internal components.
0022 # Each instance can be configured to report its metrics to one or more sinks.
0023 # Accepted values for [instance] are "master", "worker", "executor", "driver",
0024 # and "applications". A wildcard "*" can be used as an instance name, in
0025 # which case all instances will inherit the supplied property.
0026 #
0027 # Within an instance, a "source" specifies a particular set of grouped metrics.
0028 # there are two kinds of sources:
0029 # 1. Spark internal sources, like MasterSource, WorkerSource, etc, which will
0030 # collect a Spark component's internal state. Each instance is paired with a
0031 # Spark source that is added automatically.
0032 # 2. Common sources, like JvmSource, which will collect low level state.
0033 # These can be added through configuration options and are then loaded
0034 # using reflection.
0035 #
0036 # A "sink" specifies where metrics are delivered to. Each instance can be
0037 # assigned one or more sinks.
0038 #
0039 # The sink|source field specifies whether the property relates to a sink or
0040 # source.
0041 #
0042 # The [name] field specifies the name of source or sink.
0043 #
0044 # The [options] field is the specific property of this source or sink. The
0045 # source or sink is responsible for parsing this property.
0046 #
0047 # Notes:
0048 # 1. To add a new sink, set the "class" option to a fully qualified class
0049 # name (see examples below).
0050 # 2. Some sinks involve a polling period. The minimum allowed polling period
0051 # is 1 second.
0052 # 3. Wildcard properties can be overridden by more specific properties.
0053 # For example, master.sink.console.period takes precedence over
0054 # *.sink.console.period.
0055 # 4. A metrics specific configuration
0056 # "spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties" should be
0057 # added to Java properties using -Dspark.metrics.conf=xxx if you want to
0058 # customize metrics system. You can also put the file in ${SPARK_HOME}/conf
0059 # and it will be loaded automatically.
0060 # 5. The MetricsServlet sink is added by default as a sink in the master,
0061 # worker and driver, and you can send HTTP requests to the "/metrics/json"
0062 # endpoint to get a snapshot of all the registered metrics in JSON format.
0063 # For master, requests to the "/metrics/master/json" and
0064 # "/metrics/applications/json" endpoints can be sent separately to get
0065 # metrics snapshots of the master instance and applications. This
0066 # MetricsServlet does not have to be configured.
0067 # 6. The metrics system can also be configured using Spark configuration
0068 # parameters. The relevant parameter names are formed by adding the
0069 # prefix "spark.metrics.conf." to the configuration entries detailed in
0070 # this file (see examples below).
0071
0072 ## List of available common sources and their properties.
0073
0074 # org.apache.spark.metrics.source.JvmSource
0075 # Note: Currently, JvmSource is the only available common source.
0076 # It can be added to an instance by setting the "class" option to its
0077 # fully qualified class name (see examples below).
0078
0079 ## List of available sinks and their properties.
0080
0081 # org.apache.spark.metrics.sink.ConsoleSink
0082 # Name: Default: Description:
0083 # period 10 Poll period
0084 # unit seconds Unit of the poll period
0085
0086 # org.apache.spark.metrics.sink.CSVSink
0087 # Name: Default: Description:
0088 # period 10 Poll period
0089 # unit seconds Unit of the poll period
0090 # directory /tmp Where to store CSV files
0091
0092 # org.apache.spark.metrics.sink.GangliaSink
0093 # Name: Default: Description:
0094 # host NONE Hostname or multicast group of the Ganglia server,
0095 # must be set
0096 # port NONE Port of the Ganglia server(s), must be set
0097 # period 10 Poll period
0098 # unit seconds Unit of the poll period
0099 # ttl 1 TTL of messages sent by Ganglia
0100 # dmax 0 Lifetime in seconds of metrics (0 never expired)
0101 # mode multicast Ganglia network mode ('unicast' or 'multicast')
0102
0103 # org.apache.spark.metrics.sink.JmxSink
0104
0105 # org.apache.spark.metrics.sink.MetricsServlet
0106 # Name: Default: Description:
0107 # path VARIES* Path prefix from the web server root
0108 # sample false Whether to show entire set of samples for histograms
0109 # ('false' or 'true')
0110 #
0111 # * Default path is /metrics/json for all instances except the master. The
0112 # master has two paths:
0113 # /metrics/applications/json # App information
0114 # /metrics/master/json # Master information
0115
0116 # org.apache.spark.metrics.sink.PrometheusServlet
0117 # Name: Default: Description:
0118 # path VARIES* Path prefix from the web server root
0119 #
0120 # * Default path is /metrics/prometheus for all instances except the master. The
0121 # master has two paths:
0122 # /metrics/applications/prometheus # App information
0123 # /metrics/master/prometheus # Master information
0124
0125 # org.apache.spark.metrics.sink.GraphiteSink
0126 # Name: Default: Description:
0127 # host NONE Hostname of the Graphite server, must be set
0128 # port NONE Port of the Graphite server, must be set
0129 # period 10 Poll period
0130 # unit seconds Unit of the poll period
0131 # prefix EMPTY STRING Prefix to prepend to every metric's name
0132 # protocol tcp Protocol ("tcp" or "udp") to use
0133 # regex NONE Optional filter to send only metrics matching this regex string
0134
0135 # org.apache.spark.metrics.sink.StatsdSink
0136 # Name: Default: Description:
0137 # host 127.0.0.1 Hostname or IP of StatsD server
0138 # port 8125 Port of StatsD server
0139 # period 10 Poll period
0140 # unit seconds Units of poll period
0141 # prefix EMPTY STRING Prefix to prepend to metric name
0142
0143 ## Examples
0144 # Enable JmxSink for all instances by class name
0145 #*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink
0146
0147 # Enable ConsoleSink for all instances by class name
0148 #*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink
0149
0150 # Enable StatsdSink for all instances by class name
0151 #*.sink.statsd.class=org.apache.spark.metrics.sink.StatsdSink
0152 #*.sink.statsd.prefix=spark
0153
0154 # Polling period for the ConsoleSink
0155 #*.sink.console.period=10
0156 # Unit of the polling period for the ConsoleSink
0157 #*.sink.console.unit=seconds
0158
0159 # Polling period for the ConsoleSink specific for the master instance
0160 #master.sink.console.period=15
0161 # Unit of the polling period for the ConsoleSink specific for the master
0162 # instance
0163 #master.sink.console.unit=seconds
0164
0165 # Enable CsvSink for all instances by class name
0166 #*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink
0167
0168 # Polling period for the CsvSink
0169 #*.sink.csv.period=1
0170 # Unit of the polling period for the CsvSink
0171 #*.sink.csv.unit=minutes
0172
0173 # Polling directory for CsvSink
0174 #*.sink.csv.directory=/tmp/
0175
0176 # Polling period for the CsvSink specific for the worker instance
0177 #worker.sink.csv.period=10
0178 # Unit of the polling period for the CsvSink specific for the worker instance
0179 #worker.sink.csv.unit=minutes
0180
0181 # Enable Slf4jSink for all instances by class name
0182 #*.sink.slf4j.class=org.apache.spark.metrics.sink.Slf4jSink
0183
0184 # Polling period for the Slf4JSink
0185 #*.sink.slf4j.period=1
0186 # Unit of the polling period for the Slf4jSink
0187 #*.sink.slf4j.unit=minutes
0188
0189 # Example configuration for Graphite sink
0190 #*.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink
0191 #*.sink.graphite.host=<graphiteEndPoint_hostName>
0192 #*.sink.graphite.port=<listening_port>
0193 #*.sink.graphite.period=10
0194 #*.sink.graphite.unit=seconds
0195 #*.sink.graphite.prefix=<optional_value>
0196
0197 # Enable JvmSource for instance master, worker, driver and executor
0198 #master.source.jvm.class=org.apache.spark.metrics.source.JvmSource
0199
0200 #worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource
0201
0202 #driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource
0203
0204 #executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource
0205
0206 # Example configuration for PrometheusServlet
0207 #*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet
0208 #*.sink.prometheusServlet.path=/metrics/prometheus
0209 #master.sink.prometheusServlet.path=/metrics/master/prometheus
0210 #applications.sink.prometheusServlet.path=/metrics/applications/prometheus
