[Spark] Scala 단어 수 세기 예제

package com.ex

import org.apache.spark.{SparkConf, SparkContext}

/**
 * Created by moonsun on 17. 3. 14..
 */
object WordCount {

  def main(args: Array[String]) {

    val conf = new SparkConf().setMaster("local").setAppName("WordCount")
    val sc = new SparkContext(conf)

    val rdd = sc.textFile("/Users/moonsun/spark/spark2/README.md")

    val words = rdd.flatMap(line => line.split(" "))

    //words.foreach { println }
    var i = 1
    for(w <- words) {
      println( " " + i + ":" + w)
      i += 1
    }

    println("reduce before count = " + words.count())

    val rdWords = words.map(word => (word, 1)).reduceByKey{case (x,y) => x + y}

    i = 1
    for(w <- rdWords) {
      println(" " + i + ":" + w._1 + "(" + w._2 + ")")
      i += 1
    }

    println("reduce after count = " + rdWords.count())

  }

}