1import org.apache.log4j.{Level, Logger} 2import org.apache.spark.{SparkConf, SparkContext} 3import org.apache.spark.mllib.classification.{NaiveBayes, NaiveBayesModel} 4import org.apache.spark.mllib.linalg.Vectors 5import org.apache.spark.mllib.regression.LabeledPoint 6 7 8//数据格式：类别，特征1 特征2 特征3 9//0,1 0 1 10//1,0 2 0 11 12object tmp_naive_bayes { 13 14 def main(args: Array[String]){ 15 //1、构建spark对象 16 val conf = new SparkConf().setAppName("naive_bayes").setMaster("local") 17 val sc = new SparkContext(conf) 18 Logger.getRootLogger.setLevel(Level.WARN) 19 20 21 //2、读取数据样本 22 val data = sc.textFile("C://Users/wpguoc/Desktop/Spark MLlib/navie_bayes_data.txt") 23 val parsedData = data.map{ line => 24 val parts = line.split(',') 25 LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble))) 26 } 27 28 29 //3、样本数据划分训练样本和测试样本 30 val splits = parsedData.randomSplit(Array(0.6, 0.4), seed = 11L) 31 val tran = splits(0) 32 val test = splits(1) 33 34 35 //4、新建贝叶斯分类模型，并训练 36 val model = NaiveBayes.train(tran, lambda = 1.0, modelType = "multinomial") 37 38 39 //5、对测试样本进行测试 40 val predictionAndLabel = test.map(p => (model.predict(p.features), p.label)) 41 val print_predict = predictionAndLabel.take(20) 42 println("贝叶斯分类结果：" + "\n" + "prediction" + "\t" + "label") 43 for(i <- 0 to print_predict.length - 1){ 44 println(print_predict(i)._1+"\t\t\t"+print_predict(i)._2) 45 } 46 47 val accuracy = 1.0*predictionAndLabel.filter(x =>x._1 == x._2).count() / test.count() 48 println("贝叶斯分类精度" + "\n" + "accuracy: Double = " + accuracy) 49 50 51 //6、保存模型 52 val ModelPath = "C://Users/wpguoc/Desktop/Spark_MLlib/" 53 model.save(sc, ModelPath) 54 val sameModel = NaiveBayesModel.load(sc, ModelPath) 55 56 } 57 58}
运行过程及结果
1Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 218/12/19 17:32:04 INFO SparkContext: Running Spark version 1.6.3 318/12/19 17:32:04 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 418/12/19 17:32:05 INFO SecurityManager: Changing view acls to: wpguoc 518/12/19 17:32:05 INFO SecurityManager: Changing modify acls to: wpguoc 618/12/19 17:32:05 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(wpguoc); users with modify permissions: Set(wpguoc) 718/12/19 17:32:05 INFO Utils: Successfully started service 'sparkDriver' on port 63424. 818/12/19 17:32:06 INFO Slf4jLogger: Slf4jLogger started 918/12/19 17:32:06 INFO Remoting: Starting remoting 1018/12/19 17:32:06 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@192.168.66.80:63437] 1118/12/19 17:32:06 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 63437. 1218/12/19 17:32:06 INFO SparkEnv: Registering MapOutputTracker 1318/12/19 17:32:06 INFO SparkEnv: Registering BlockManagerMaster 1418/12/19 17:32:06 INFO DiskBlockManager: Created local directory at C:\Users\wpguoc\AppData\Local\Temp\blockmgr-4d798e34-90b0-4ee9-a811-586a893f4818 1518/12/19 17:32:06 INFO MemoryStore: MemoryStore started with capacity 1127.3 MB 1618/12/19 17:32:06 INFO SparkEnv: Registering OutputCommitCoordinator 1718/12/19 17:32:06 INFO Utils: Successfully started service 'SparkUI' on port 4040. 1818/12/19 17:32:06 INFO SparkUI: Started SparkUI at http://192.168.66.80:4040 1918/12/19 17:32:06 INFO Executor: Starting executor ID driver on host localhost 2018/12/19 17:32:06 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 63444. 2118/12/19 17:32:06 INFO NettyBlockTransferService: Server created on 63444 2218/12/19 17:32:06 INFO BlockManagerMaster: Trying to register BlockManager 2318/12/19 17:32:06 INFO BlockManagerMasterEndpoint: Registering block manager localhost:63444 with 1127.3 MB RAM, BlockManagerId(driver, localhost, 63444) 2418/12/19 17:32:06 INFO BlockManagerMaster: Registered BlockManager 2518/12/19 17:32:10 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 2618/12/19 17:32:10 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS 27贝叶斯分类结果： 28prediction label 290.0 0.0 300.0 0.0 312.0 2.0 322.0 2.0 332.0 2.0 34贝叶斯分类精度 35accuracy: Double = 1.0 36SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder". 37SLF4J: Defaulting to no-operation (NOP) logger implementation 38SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details. 3918/12/19 17:32:14 WARN ParquetRecordReader: Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl 40 41Process finished with exit code 0