spark11-sparkSQL 实现wordcount

小朋友,你是否有很多问号?

已于 2025-09-09 18:20:47 修改

阅读量143

点赞数 1

CC 4.0 BY-SA版权

分类专栏： spark 文章标签： spark

于 2025-09-09 18:19:38 首次发布

本文链接：https://blog.csdn.net/oracle8090/article/details/151368630

spark 专栏收录该内容

14 篇文章

订阅专栏

文件名为 words.txt

spark scala
hello spark
word word

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}

object SparkSqlHelloWorld {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setMaster("local[1]").setAppName("sql")
    val session: SparkSession = SparkSession.builder().config(conf).getOrCreate()
    val sc: SparkContext = session.sparkContext
    val sqlContext: SQLContext = session.sqlContext
    val rdd: RDD[String] = sc.textFile("/Users/yolo/IdeaProjects/SparkStudy/src/main/scala/works.txt")
    import session.implicits._
    val df: DataFrame = rdd.flatMap(_.split(" ")).toDF("wd")
    df.createTempView("words")
    val sql = sqlContext.sql("select wd,count(1) from words group by wd")
    sql.show()
    sql.rdd.repartition(1).saveAsTextFile("/Users/yolo/IdeaProjects/SparkStudy/src/main/scala/worksCnt")
    session.close()
  }
}