0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018 package org.apache.spark.examples.ml;
0019
0020 import org.apache.spark.ml.clustering.LDA;
0021 import org.apache.spark.ml.clustering.LDAModel;
0022 import org.apache.spark.sql.Dataset;
0023 import org.apache.spark.sql.Row;
0024 import org.apache.spark.sql.SparkSession;
0025
0026
0027
0028
0029
0030
0031
0032
0033
0034 public class JavaLDAExample {
0035
0036 public static void main(String[] args) {
0037
0038 SparkSession spark = SparkSession
0039 .builder()
0040 .appName("JavaLDAExample")
0041 .getOrCreate();
0042
0043
0044
0045 Dataset<Row> dataset = spark.read().format("libsvm")
0046 .load("data/mllib/sample_lda_libsvm_data.txt");
0047
0048
0049 LDA lda = new LDA().setK(10).setMaxIter(10);
0050 LDAModel model = lda.fit(dataset);
0051
0052 double ll = model.logLikelihood(dataset);
0053 double lp = model.logPerplexity(dataset);
0054 System.out.println("The lower bound on the log likelihood of the entire corpus: " + ll);
0055 System.out.println("The upper bound on perplexity: " + lp);
0056
0057
0058 Dataset<Row> topics = model.describeTopics(3);
0059 System.out.println("The topics described by their top-weighted terms:");
0060 topics.show(false);
0061
0062
0063 Dataset<Row> transformed = model.transform(dataset);
0064 transformed.show(false);
0065
0066
0067 spark.stop();
0068 }
0069 }