0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018 """
0019 NaiveBayes Example.
0020
0021 Usage:
0022 `spark-submit --master local[4] examples/src/main/python/mllib/naive_bayes_example.py`
0023 """
0024
0025 from __future__ import print_function
0026
0027 import shutil
0028
0029 from pyspark import SparkContext
0030
0031 from pyspark.mllib.classification import NaiveBayes, NaiveBayesModel
0032 from pyspark.mllib.util import MLUtils
0033
0034
0035
0036
0037 if __name__ == "__main__":
0038
0039 sc = SparkContext(appName="PythonNaiveBayesExample")
0040
0041
0042
0043 data = MLUtils.loadLibSVMFile(sc, "data/mllib/sample_libsvm_data.txt")
0044
0045
0046 training, test = data.randomSplit([0.6, 0.4])
0047
0048
0049 model = NaiveBayes.train(training, 1.0)
0050
0051
0052 predictionAndLabel = test.map(lambda p: (model.predict(p.features), p.label))
0053 accuracy = 1.0 * predictionAndLabel.filter(lambda pl: pl[0] == pl[1]).count() / test.count()
0054 print('model accuracy {}'.format(accuracy))
0055
0056
0057 output_dir = 'target/tmp/myNaiveBayesModel'
0058 shutil.rmtree(output_dir, ignore_errors=True)
0059 model.save(sc, output_dir)
0060 sameModel = NaiveBayesModel.load(sc, output_dir)
0061 predictionAndLabel = test.map(lambda p: (sameModel.predict(p.features), p.label))
0062 accuracy = 1.0 * predictionAndLabel.filter(lambda pl: pl[0] == pl[1]).count() / test.count()
0063 print('sameModel accuracy {}'.format(accuracy))
0064
0065