python/ml/one_vs_rest_example.py

0001 #
0002 # Licensed to the Apache Software Foundation (ASF) under one or more
0003 # contributor license agreements.  See the NOTICE file distributed with
0004 # this work for additional information regarding copyright ownership.
0005 # The ASF licenses this file to You under the Apache License, Version 2.0
0006 # (the "License"); you may not use this file except in compliance with
0007 # the License.  You may obtain a copy of the License at
0008 #
0009 #    http://www.apache.org/licenses/LICENSE-2.0
0010 #
0011 # Unless required by applicable law or agreed to in writing, software
0012 # distributed under the License is distributed on an "AS IS" BASIS,
0013 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014 # See the License for the specific language governing permissions and
0015 # limitations under the License.
0016 #
0017
0018 """
0019 An example of Multiclass to Binary Reduction with One Vs Rest,
0020 using Logistic Regression as the base classifier.
0021 Run with:
0022   bin/spark-submit examples/src/main/python/ml/one_vs_rest_example.py
0023 """
0024 from __future__ import print_function
0025
0026 # $example on$
0027 from pyspark.ml.classification import LogisticRegression, OneVsRest
0028 from pyspark.ml.evaluation import MulticlassClassificationEvaluator
0029 # $example off$
0030 from pyspark.sql import SparkSession
0031
0032 if __name__ == "__main__":
0033     spark = SparkSession \
0034         .builder \
0035         .appName("OneVsRestExample") \
0036         .getOrCreate()
0037
0038     # $example on$
0039     # load data file.
0040     inputData = spark.read.format("libsvm") \
0041         .load("data/mllib/sample_multiclass_classification_data.txt")
0042
0043     # generate the train/test split.
0044     (train, test) = inputData.randomSplit([0.8, 0.2])
0045
0046     # instantiate the base classifier.
0047     lr = LogisticRegression(maxIter=10, tol=1E-6, fitIntercept=True)
0048
0049     # instantiate the One Vs Rest Classifier.
0050     ovr = OneVsRest(classifier=lr)
0051
0052     # train the multiclass model.
0053     ovrModel = ovr.fit(train)
0054
0055     # score the model on test data.
0056     predictions = ovrModel.transform(test)
0057
0058     # obtain evaluator.
0059     evaluator = MulticlassClassificationEvaluator(metricName="accuracy")
0060
0061     # compute the classification error on test data.
0062     accuracy = evaluator.evaluate(predictions)
0063     print("Test Error = %g" % (1.0 - accuracy))
0064     # $example off$
0065
0066     spark.stop()