spark/launcher/CommandBuilderUtils.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.launcher;
0019
0020 import java.io.File;
0021 import java.util.ArrayList;
0022 import java.util.List;
0023 import java.util.Map;
0024
0025 /**
0026  * Helper methods for command builders.
0027  */
0028 class CommandBuilderUtils {
0029
0030   static final String DEFAULT_MEM = "1g";
0031   static final String DEFAULT_PROPERTIES_FILE = "spark-defaults.conf";
0032   static final String ENV_SPARK_HOME = "SPARK_HOME";
0033
0034   /** Returns whether the given string is null or empty. */
0035   static boolean isEmpty(String s) {
0036     return s == null || s.isEmpty();
0037   }
0038
0039   /** Joins a list of strings using the given separator. */
0040   static String join(String sep, String... elements) {
0041     StringBuilder sb = new StringBuilder();
0042     for (String e : elements) {
0043       if (e != null) {
0044         if (sb.length() > 0) {
0045           sb.append(sep);
0046         }
0047         sb.append(e);
0048       }
0049     }
0050     return sb.toString();
0051   }
0052
0053   /** Joins a list of strings using the given separator. */
0054   static String join(String sep, Iterable<String> elements) {
0055     StringBuilder sb = new StringBuilder();
0056     for (String e : elements) {
0057       if (e != null) {
0058         if (sb.length() > 0) {
0059           sb.append(sep);
0060         }
0061         sb.append(e);
0062       }
0063     }
0064     return sb.toString();
0065   }
0066
0067   /**
0068    * Returns the first non-empty value mapped to the given key in the given maps, or null otherwise.
0069    */
0070   static String firstNonEmptyValue(String key, Map<?, ?>... maps) {
0071     for (Map<?, ?> map : maps) {
0072       String value = (String) map.get(key);
0073       if (!isEmpty(value)) {
0074         return value;
0075       }
0076     }
0077     return null;
0078   }
0079
0080   /** Returns the first non-empty, non-null string in the given list, or null otherwise. */
0081   static String firstNonEmpty(String... candidates) {
0082     for (String s : candidates) {
0083       if (!isEmpty(s)) {
0084         return s;
0085       }
0086     }
0087     return null;
0088   }
0089
0090   /** Returns the name of the env variable that holds the native library path. */
0091   static String getLibPathEnvName() {
0092     if (isWindows()) {
0093       return "PATH";
0094     }
0095
0096     String os = System.getProperty("os.name");
0097     if (os.startsWith("Mac OS X")) {
0098       return "DYLD_LIBRARY_PATH";
0099     } else {
0100       return "LD_LIBRARY_PATH";
0101     }
0102   }
0103
0104   /** Returns whether the OS is Windows. */
0105   static boolean isWindows() {
0106     String os = System.getProperty("os.name");
0107     return os.startsWith("Windows");
0108   }
0109
0110   /**
0111    * Updates the user environment, appending the given pathList to the existing value of the given
0112    * environment variable (or setting it if it hasn't yet been set).
0113    */
0114   static void mergeEnvPathList(Map<String, String> userEnv, String envKey, String pathList) {
0115     if (!isEmpty(pathList)) {
0116       String current = firstNonEmpty(userEnv.get(envKey), System.getenv(envKey));
0117       userEnv.put(envKey, join(File.pathSeparator, current, pathList));
0118     }
0119   }
0120
0121   /**
0122    * Parse a string as if it were a list of arguments, following bash semantics.
0123    * For example:
0124    *
0125    * Input: "\"ab cd\" efgh 'i \" j'"
0126    * Output: [ "ab cd", "efgh", "i \" j" ]
0127    */
0128   static List<String> parseOptionString(String s) {
0129     List<String> opts = new ArrayList<>();
0130     StringBuilder opt = new StringBuilder();
0131     boolean inOpt = false;
0132     boolean inSingleQuote = false;
0133     boolean inDoubleQuote = false;
0134     boolean escapeNext = false;
0135
0136     // This is needed to detect when a quoted empty string is used as an argument ("" or '').
0137     boolean hasData = false;
0138
0139     for (int i = 0; i < s.length(); i++) {
0140       int c = s.codePointAt(i);
0141       if (escapeNext) {
0142         opt.appendCodePoint(c);
0143         escapeNext = false;
0144       } else if (inOpt) {
0145         switch (c) {
0146         case '\\':
0147           if (inSingleQuote) {
0148             opt.appendCodePoint(c);
0149           } else {
0150             escapeNext = true;
0151           }
0152           break;
0153         case '\'':
0154           if (inDoubleQuote) {
0155             opt.appendCodePoint(c);
0156           } else {
0157             inSingleQuote = !inSingleQuote;
0158           }
0159           break;
0160         case '"':
0161           if (inSingleQuote) {
0162             opt.appendCodePoint(c);
0163           } else {
0164             inDoubleQuote = !inDoubleQuote;
0165           }
0166           break;
0167         default:
0168           if (!Character.isWhitespace(c) || inSingleQuote || inDoubleQuote) {
0169             opt.appendCodePoint(c);
0170           } else {
0171             opts.add(opt.toString());
0172             opt.setLength(0);
0173             inOpt = false;
0174             hasData = false;
0175           }
0176         }
0177       } else {
0178         switch (c) {
0179         case '\'':
0180           inSingleQuote = true;
0181           inOpt = true;
0182           hasData = true;
0183           break;
0184         case '"':
0185           inDoubleQuote = true;
0186           inOpt = true;
0187           hasData = true;
0188           break;
0189         case '\\':
0190           escapeNext = true;
0191           inOpt = true;
0192           hasData = true;
0193           break;
0194         default:
0195           if (!Character.isWhitespace(c)) {
0196             inOpt = true;
0197             hasData = true;
0198             opt.appendCodePoint(c);
0199           }
0200         }
0201       }
0202     }
0203
0204     checkArgument(!inSingleQuote && !inDoubleQuote && !escapeNext, "Invalid option string: %s", s);
0205     if (hasData) {
0206       opts.add(opt.toString());
0207     }
0208     return opts;
0209   }
0210
0211   /** Throws IllegalArgumentException if the given object is null. */
0212   static void checkNotNull(Object o, String arg) {
0213     if (o == null) {
0214       throw new IllegalArgumentException(String.format("'%s' must not be null.", arg));
0215     }
0216   }
0217
0218   /** Throws IllegalArgumentException with the given message if the check is false. */
0219   static void checkArgument(boolean check, String msg, Object... args) {
0220     if (!check) {
0221       throw new IllegalArgumentException(String.format(msg, args));
0222     }
0223   }
0224
0225   /** Throws IllegalStateException with the given message if the check is false. */
0226   static void checkState(boolean check, String msg, Object... args) {
0227     if (!check) {
0228       throw new IllegalStateException(String.format(msg, args));
0229     }
0230   }
0231
0232   /**
0233    * Quote a command argument for a command to be run by a Windows batch script, if the argument
0234    * needs quoting. Arguments only seem to need quotes in batch scripts if they have certain
0235    * special characters, some of which need extra (and different) escaping.
0236    *
0237    *  For example:
0238    *    original single argument: ab="cde fgh"
0239    *    quoted: "ab^=""cde fgh"""
0240    */
0241   static String quoteForBatchScript(String arg) {
0242
0243     boolean needsQuotes = false;
0244     for (int i = 0; i < arg.length(); i++) {
0245       int c = arg.codePointAt(i);
0246       if (Character.isWhitespace(c) || c == '"' || c == '=' || c == ',' || c == ';') {
0247         needsQuotes = true;
0248         break;
0249       }
0250     }
0251     if (!needsQuotes) {
0252       return arg;
0253     }
0254     StringBuilder quoted = new StringBuilder();
0255     quoted.append("\"");
0256     for (int i = 0; i < arg.length(); i++) {
0257       int cp = arg.codePointAt(i);
0258       switch (cp) {
0259       case '"':
0260         quoted.append('"');
0261         break;
0262
0263       default:
0264         break;
0265       }
0266       quoted.appendCodePoint(cp);
0267     }
0268     if (arg.codePointAt(arg.length() - 1) == '\\') {
0269       quoted.append("\\");
0270     }
0271     quoted.append("\"");
0272     return quoted.toString();
0273   }
0274
0275   /**
0276    * Quotes a string so that it can be used in a command string.
0277    * Basically, just add simple escapes. E.g.:
0278    *    original single argument : ab "cd" ef
0279    *    after: "ab \"cd\" ef"
0280    *
0281    * This can be parsed back into a single argument by python's "shlex.split()" function.
0282    */
0283   static String quoteForCommandString(String s) {
0284     StringBuilder quoted = new StringBuilder().append('"');
0285     for (int i = 0; i < s.length(); i++) {
0286       int cp = s.codePointAt(i);
0287       if (cp == '"' || cp == '\\') {
0288         quoted.appendCodePoint('\\');
0289       }
0290       quoted.appendCodePoint(cp);
0291     }
0292     return quoted.append('"').toString();
0293   }
0294
0295   /**
0296    * Get the major version of the java version string supplied. This method
0297    * accepts any JEP-223-compliant strings (9-ea, 9+100), as well as legacy
0298    * version strings such as 1.7.0_79
0299    */
0300   static int javaMajorVersion(String javaVersion) {
0301     String[] version = javaVersion.split("[+.\\-]+");
0302     int major = Integer.parseInt(version[0]);
0303     // if major > 1, we're using the JEP-223 version string, e.g., 9-ea, 9+120
0304     // otherwise the second number is the major version
0305     if (major > 1) {
0306       return major;
0307     } else {
0308       return Integer.parseInt(version[1]);
0309     }
0310   }
0311
0312   /**
0313    * Find the location of the Spark jars dir, depending on whether we're looking at a build
0314    * or a distribution directory.
0315    */
0316   static String findJarsDir(String sparkHome, String scalaVersion, boolean failIfNotFound) {
0317     // TODO: change to the correct directory once the assembly build is changed.
0318     File libdir = new File(sparkHome, "jars");
0319     if (!libdir.isDirectory()) {
0320       libdir = new File(sparkHome, String.format("assembly/target/scala-%s/jars", scalaVersion));
0321       if (!libdir.isDirectory()) {
0322         checkState(!failIfNotFound,
0323           "Library directory '%s' does not exist; make sure Spark is built.",
0324           libdir.getAbsolutePath());
0325         return null;
0326       }
0327     }
0328     return libdir.getAbsolutePath();
0329   }
0330
0331 }