spark/launcher/SparkSubmitOptionParser.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.launcher;
0019
0020 import java.util.List;
0021 import java.util.regex.Matcher;
0022 import java.util.regex.Pattern;
0023
0024 /**
0025  * Parser for spark-submit command line options.
0026  * <p>
0027  * This class encapsulates the parsing code for spark-submit command line options, so that there
0028  * is a single list of options that needs to be maintained (well, sort of, but it makes it harder
0029  * to break things).
0030  */
0031 class SparkSubmitOptionParser {
0032
0033   // The following constants define the "main" name for the available options. They're defined
0034   // to avoid copy & paste of the raw strings where they're needed.
0035   //
0036   // The fields are not static so that they're exposed to Scala code that uses this class. See
0037   // SparkSubmitArguments.scala. That is also why this class is not abstract - to allow code to
0038   // easily use these constants without having to create dummy implementations of this class.
0039   protected final String CLASS = "--class";
0040   protected final String CONF = "--conf";
0041   protected final String DEPLOY_MODE = "--deploy-mode";
0042   protected final String DRIVER_CLASS_PATH = "--driver-class-path";
0043   protected final String DRIVER_CORES = "--driver-cores";
0044   protected final String DRIVER_JAVA_OPTIONS =  "--driver-java-options";
0045   protected final String DRIVER_LIBRARY_PATH = "--driver-library-path";
0046   protected final String DRIVER_MEMORY = "--driver-memory";
0047   protected final String EXECUTOR_MEMORY = "--executor-memory";
0048   protected final String FILES = "--files";
0049   protected final String JARS = "--jars";
0050   protected final String KILL_SUBMISSION = "--kill";
0051   protected final String MASTER = "--master";
0052   protected final String NAME = "--name";
0053   protected final String PACKAGES = "--packages";
0054   protected final String PACKAGES_EXCLUDE = "--exclude-packages";
0055   protected final String PROPERTIES_FILE = "--properties-file";
0056   protected final String PROXY_USER = "--proxy-user";
0057   protected final String PY_FILES = "--py-files";
0058   protected final String REPOSITORIES = "--repositories";
0059   protected final String STATUS = "--status";
0060   protected final String TOTAL_EXECUTOR_CORES = "--total-executor-cores";
0061
0062   // Options that do not take arguments.
0063   protected final String HELP = "--help";
0064   protected final String SUPERVISE = "--supervise";
0065   protected final String USAGE_ERROR = "--usage-error";
0066   protected final String VERBOSE = "--verbose";
0067   protected final String VERSION = "--version";
0068
0069   // Standalone-only options.
0070
0071   // YARN-only options.
0072   protected final String ARCHIVES = "--archives";
0073   protected final String EXECUTOR_CORES = "--executor-cores";
0074   protected final String KEYTAB = "--keytab";
0075   protected final String NUM_EXECUTORS = "--num-executors";
0076   protected final String PRINCIPAL = "--principal";
0077   protected final String QUEUE = "--queue";
0078
0079   /**
0080    * This is the canonical list of spark-submit options. Each entry in the array contains the
0081    * different aliases for the same option; the first element of each entry is the "official"
0082    * name of the option, passed to {@link #handle(String, String)}.
0083    * <p>
0084    * Options not listed here nor in the "switch" list below will result in a call to
0085    * {@link #handleUnknown(String)}.
0086    * <p>
0087    * These two arrays are visible for tests.
0088    */
0089   final String[][] opts = {
0090     { ARCHIVES },
0091     { CLASS },
0092     { CONF, "-c" },
0093     { DEPLOY_MODE },
0094     { DRIVER_CLASS_PATH },
0095     { DRIVER_CORES },
0096     { DRIVER_JAVA_OPTIONS },
0097     { DRIVER_LIBRARY_PATH },
0098     { DRIVER_MEMORY },
0099     { EXECUTOR_CORES },
0100     { EXECUTOR_MEMORY },
0101     { FILES },
0102     { JARS },
0103     { KEYTAB },
0104     { KILL_SUBMISSION },
0105     { MASTER },
0106     { NAME },
0107     { NUM_EXECUTORS },
0108     { PACKAGES },
0109     { PACKAGES_EXCLUDE },
0110     { PRINCIPAL },
0111     { PROPERTIES_FILE },
0112     { PROXY_USER },
0113     { PY_FILES },
0114     { QUEUE },
0115     { REPOSITORIES },
0116     { STATUS },
0117     { TOTAL_EXECUTOR_CORES },
0118   };
0119
0120   /**
0121    * List of switches (command line options that do not take parameters) recognized by spark-submit.
0122    */
0123   final String[][] switches = {
0124     { HELP, "-h" },
0125     { SUPERVISE },
0126     { USAGE_ERROR },
0127     { VERBOSE, "-v" },
0128     { VERSION },
0129   };
0130
0131   /**
0132    * Parse a list of spark-submit command line options.
0133    * <p>
0134    * See SparkSubmitArguments.scala for a more formal description of available options.
0135    *
0136    * @throws IllegalArgumentException If an error is found during parsing.
0137    */
0138   protected final void parse(List<String> args) {
0139     Pattern eqSeparatedOpt = Pattern.compile("(--[^=]+)=(.+)");
0140
0141     int idx = 0;
0142     for (idx = 0; idx < args.size(); idx++) {
0143       String arg = args.get(idx);
0144       String value = null;
0145
0146       Matcher m = eqSeparatedOpt.matcher(arg);
0147       if (m.matches()) {
0148         arg = m.group(1);
0149         value = m.group(2);
0150       }
0151
0152       // Look for options with a value.
0153       String name = findCliOption(arg, opts);
0154       if (name != null) {
0155         if (value == null) {
0156           if (idx == args.size() - 1) {
0157             throw new IllegalArgumentException(
0158                 String.format("Missing argument for option '%s'.", arg));
0159           }
0160           idx++;
0161           value = args.get(idx);
0162         }
0163         if (!handle(name, value)) {
0164           break;
0165         }
0166         continue;
0167       }
0168
0169       // Look for a switch.
0170       name = findCliOption(arg, switches);
0171       if (name != null) {
0172         if (!handle(name, null)) {
0173           break;
0174         }
0175         continue;
0176       }
0177
0178       if (!handleUnknown(arg)) {
0179         break;
0180       }
0181     }
0182
0183     if (idx < args.size()) {
0184       idx++;
0185     }
0186     handleExtraArgs(args.subList(idx, args.size()));
0187   }
0188
0189   /**
0190    * Callback for when an option with an argument is parsed.
0191    *
0192    * @param opt The long name of the cli option (might differ from actual command line).
0193    * @param value The value. This will be <i>null</i> if the option does not take a value.
0194    * @return Whether to continue parsing the argument list.
0195    */
0196   protected boolean handle(String opt, String value) {
0197     throw new UnsupportedOperationException();
0198   }
0199
0200   /**
0201    * Callback for when an unrecognized option is parsed.
0202    *
0203    * @param opt Unrecognized option from the command line.
0204    * @return Whether to continue parsing the argument list.
0205    */
0206   protected boolean handleUnknown(String opt) {
0207     throw new UnsupportedOperationException();
0208   }
0209
0210   /**
0211    * Callback for remaining command line arguments after either {@link #handle(String, String)} or
0212    * {@link #handleUnknown(String)} return "false". This will be called at the end of parsing even
0213    * when there are no remaining arguments.
0214    *
0215    * @param extra List of remaining arguments.
0216    */
0217   protected void handleExtraArgs(List<String> extra) {
0218     throw new UnsupportedOperationException();
0219   }
0220
0221   private String findCliOption(String name, String[][] available) {
0222     for (String[] candidates : available) {
0223       for (String candidate : candidates) {
0224         if (candidate.equals(name)) {
0225           return candidates[0];
0226         }
0227       }
0228     }
0229     return null;
0230   }
0231
0232 }