spark/launcher/Main.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.launcher;
0019
0020 import java.io.IOException;
0021 import java.util.ArrayList;
0022 import java.util.Arrays;
0023 import java.util.HashMap;
0024 import java.util.List;
0025 import java.util.Map;
0026
0027 import static org.apache.spark.launcher.CommandBuilderUtils.*;
0028
0029 /**
0030  * Command line interface for the Spark launcher. Used internally by Spark scripts.
0031  */
0032 class Main {
0033
0034   /**
0035    * Usage: Main [class] [class args]
0036    * <p>
0037    * This CLI works in two different modes:
0038    * <ul>
0039    *   <li>"spark-submit": if <i>class</i> is "org.apache.spark.deploy.SparkSubmit", the
0040    *   {@link SparkLauncher} class is used to launch a Spark application.</li>
0041    *   <li>"spark-class": if another class is provided, an internal Spark class is run.</li>
0042    * </ul>
0043    *
0044    * This class works in tandem with the "bin/spark-class" script on Unix-like systems, and
0045    * "bin/spark-class2.cmd" batch script on Windows to execute the final command.
0046    * <p>
0047    * On Unix-like systems, the output is a list of command arguments, separated by the NULL
0048    * character. On Windows, the output is a command line suitable for direct execution from the
0049    * script.
0050    */
0051   public static void main(String[] argsArray) throws Exception {
0052     checkArgument(argsArray.length > 0, "Not enough arguments: missing class name.");
0053
0054     List<String> args = new ArrayList<>(Arrays.asList(argsArray));
0055     String className = args.remove(0);
0056
0057     boolean printLaunchCommand = !isEmpty(System.getenv("SPARK_PRINT_LAUNCH_COMMAND"));
0058     Map<String, String> env = new HashMap<>();
0059     List<String> cmd;
0060     if (className.equals("org.apache.spark.deploy.SparkSubmit")) {
0061       try {
0062         AbstractCommandBuilder builder = new SparkSubmitCommandBuilder(args);
0063         cmd = buildCommand(builder, env, printLaunchCommand);
0064       } catch (IllegalArgumentException e) {
0065         printLaunchCommand = false;
0066         System.err.println("Error: " + e.getMessage());
0067         System.err.println();
0068
0069         MainClassOptionParser parser = new MainClassOptionParser();
0070         try {
0071           parser.parse(args);
0072         } catch (Exception ignored) {
0073           // Ignore parsing exceptions.
0074         }
0075
0076         List<String> help = new ArrayList<>();
0077         if (parser.className != null) {
0078           help.add(parser.CLASS);
0079           help.add(parser.className);
0080         }
0081         help.add(parser.USAGE_ERROR);
0082         AbstractCommandBuilder builder = new SparkSubmitCommandBuilder(help);
0083         cmd = buildCommand(builder, env, printLaunchCommand);
0084       }
0085     } else {
0086       AbstractCommandBuilder builder = new SparkClassCommandBuilder(className, args);
0087       cmd = buildCommand(builder, env, printLaunchCommand);
0088     }
0089
0090     if (isWindows()) {
0091       System.out.println(prepareWindowsCommand(cmd, env));
0092     } else {
0093       // A sequence of NULL character and newline separates command-strings and others.
0094       System.out.println('\0');
0095
0096       // In bash, use NULL as the arg separator since it cannot be used in an argument.
0097       List<String> bashCmd = prepareBashCommand(cmd, env);
0098       for (String c : bashCmd) {
0099         System.out.print(c);
0100         System.out.print('\0');
0101       }
0102     }
0103   }
0104
0105   /**
0106    * Prepare spark commands with the appropriate command builder.
0107    * If printLaunchCommand is set then the commands will be printed to the stderr.
0108    */
0109   private static List<String> buildCommand(
0110       AbstractCommandBuilder builder,
0111       Map<String, String> env,
0112       boolean printLaunchCommand) throws IOException, IllegalArgumentException {
0113     List<String> cmd = builder.buildCommand(env);
0114     if (printLaunchCommand) {
0115       System.err.println("Spark Command: " + join(" ", cmd));
0116       System.err.println("========================================");
0117     }
0118     return cmd;
0119   }
0120
0121   /**
0122    * Prepare a command line for execution from a Windows batch script.
0123    *
0124    * The method quotes all arguments so that spaces are handled as expected. Quotes within arguments
0125    * are "double quoted" (which is batch for escaping a quote). This page has more details about
0126    * quoting and other batch script fun stuff: http://ss64.com/nt/syntax-esc.html
0127    */
0128   private static String prepareWindowsCommand(List<String> cmd, Map<String, String> childEnv) {
0129     StringBuilder cmdline = new StringBuilder();
0130     for (Map.Entry<String, String> e : childEnv.entrySet()) {
0131       cmdline.append(String.format("set %s=%s", e.getKey(), e.getValue()));
0132       cmdline.append(" && ");
0133     }
0134     for (String arg : cmd) {
0135       cmdline.append(quoteForBatchScript(arg));
0136       cmdline.append(" ");
0137     }
0138     return cmdline.toString();
0139   }
0140
0141   /**
0142    * Prepare the command for execution from a bash script. The final command will have commands to
0143    * set up any needed environment variables needed by the child process.
0144    */
0145   private static List<String> prepareBashCommand(List<String> cmd, Map<String, String> childEnv) {
0146     if (childEnv.isEmpty()) {
0147       return cmd;
0148     }
0149
0150     List<String> newCmd = new ArrayList<>();
0151     newCmd.add("env");
0152
0153     for (Map.Entry<String, String> e : childEnv.entrySet()) {
0154       newCmd.add(String.format("%s=%s", e.getKey(), e.getValue()));
0155     }
0156     newCmd.addAll(cmd);
0157     return newCmd;
0158   }
0159
0160   /**
0161    * A parser used when command line parsing fails for spark-submit. It's used as a best-effort
0162    * at trying to identify the class the user wanted to invoke, since that may require special
0163    * usage strings (handled by SparkSubmitArguments).
0164    */
0165   private static class MainClassOptionParser extends SparkSubmitOptionParser {
0166
0167     String className;
0168
0169     @Override
0170     protected boolean handle(String opt, String value) {
0171       if (CLASS.equals(opt)) {
0172         className = value;
0173       }
0174       return false;
0175     }
0176
0177     @Override
0178     protected boolean handleUnknown(String opt) {
0179       return false;
0180     }
0181
0182     @Override
0183     protected void handleExtraArgs(List<String> extra) {
0184
0185     }
0186
0187   }
0188
0189 }