spark/examples/JavaLogQuery.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.examples;
0019
0020 import scala.Tuple2;
0021 import scala.Tuple3;
0022
0023 import org.apache.spark.api.java.JavaPairRDD;
0024 import org.apache.spark.api.java.JavaRDD;
0025 import org.apache.spark.api.java.JavaSparkContext;
0026 import org.apache.spark.sql.SparkSession;
0027
0028 import java.io.Serializable;
0029 import java.util.Arrays;
0030 import java.util.List;
0031 import java.util.regex.Matcher;
0032 import java.util.regex.Pattern;
0033
0034 /**
0035  * Executes a roll up-style query against Apache logs.
0036  *
0037  * Usage: JavaLogQuery [logFile]
0038  */
0039 public final class JavaLogQuery {
0040
0041   public static final List<String> exampleApacheLogs = Arrays.asList(
0042     "10.10.10.10 - \"FRED\" [18/Jan/2013:17:56:07 +1100] \"GET http://images.com/2013/Generic.jpg " +
0043       "HTTP/1.1\" 304 315 \"http://referall.com/\" \"Mozilla/4.0 (compatible; MSIE 7.0; " +
0044       "Windows NT 5.1; GTB7.4; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648; " +
0045       ".NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR " +
0046       "3.5.30729; Release=ARP)\" \"UD-1\" - \"image/jpeg\" \"whatever\" 0.350 \"-\" - \"\" 265 923 934 \"\" " +
0047       "62.24.11.25 images.com 1358492167 - Whatup",
0048     "10.10.10.10 - \"FRED\" [18/Jan/2013:18:02:37 +1100] \"GET http://images.com/2013/Generic.jpg " +
0049       "HTTP/1.1\" 304 306 \"http:/referall.com\" \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; " +
0050       "GTB7.4; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648; .NET CLR " +
0051       "3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR  " +
0052       "3.5.30729; Release=ARP)\" \"UD-1\" - \"image/jpeg\" \"whatever\" 0.352 \"-\" - \"\" 256 977 988 \"\" " +
0053       "0 73.23.2.15 images.com 1358492557 - Whatup");
0054
0055   public static final Pattern apacheLogRegex = Pattern.compile(
0056     "^([\\d.]+) (\\S+) (\\S+) \\[([\\w\\d:/]+\\s[+\\-]\\d{4})\\] \"(.+?)\" (\\d{3}) ([\\d\\-]+) \"([^\"]+)\" \"([^\"]+)\".*");
0057
0058   /** Tracks the total query count and number of aggregate bytes for a particular group. */
0059   public static class Stats implements Serializable {
0060
0061     private final int count;
0062     private final int numBytes;
0063
0064     public Stats(int count, int numBytes) {
0065       this.count = count;
0066       this.numBytes = numBytes;
0067     }
0068     public Stats merge(Stats other) {
0069       return new Stats(count + other.count, numBytes + other.numBytes);
0070     }
0071
0072     public String toString() {
0073       return String.format("bytes=%s\tn=%s", numBytes, count);
0074     }
0075   }
0076
0077   public static Tuple3<String, String, String> extractKey(String line) {
0078     Matcher m = apacheLogRegex.matcher(line);
0079     if (m.find()) {
0080       String ip = m.group(1);
0081       String user = m.group(3);
0082       String query = m.group(5);
0083       if (!user.equalsIgnoreCase("-")) {
0084         return new Tuple3<>(ip, user, query);
0085       }
0086     }
0087     return new Tuple3<>(null, null, null);
0088   }
0089
0090   public static Stats extractStats(String line) {
0091     Matcher m = apacheLogRegex.matcher(line);
0092     if (m.find()) {
0093       int bytes = Integer.parseInt(m.group(7));
0094       return new Stats(1, bytes);
0095     } else {
0096       return new Stats(1, 0);
0097     }
0098   }
0099
0100   public static void main(String[] args) {
0101     SparkSession spark = SparkSession
0102       .builder()
0103       .appName("JavaLogQuery")
0104       .getOrCreate();
0105
0106     JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());
0107
0108     JavaRDD<String> dataSet = (args.length == 1) ? jsc.textFile(args[0]) : jsc.parallelize(exampleApacheLogs);
0109
0110     JavaPairRDD<Tuple3<String, String, String>, Stats> extracted =
0111         dataSet.mapToPair(s -> new Tuple2<>(extractKey(s), extractStats(s)));
0112
0113     JavaPairRDD<Tuple3<String, String, String>, Stats> counts = extracted.reduceByKey(Stats::merge);
0114
0115     List<Tuple2<Tuple3<String, String, String>, Stats>> output = counts.collect();
0116     for (Tuple2<?,?> t : output) {
0117       System.out.println(t._1() + "\t" + t._2());
0118     }
0119     spark.stop();
0120   }
0121 }