Spark DataFrames and SQL

docker run -v /home/etadm/docker/spark/e2spkv01:/home:rw
-p 8088:8088 -p 8042:8042
-h sandbox
-it sequenceiq/spark:1.6.0 bash
2 . 5

spark-shell
--master local
--jars /home/e2-spk-s02/jars/spark-csv_2.10-1.4.0.jar
,/home/e2-spk-s02/jars/commons-csv-1.1.jar
2 . 6

// Create a object container
case class Word(text: String)
val fileName = "README.md"
val docs = sc.textFile(fileName)
val lower = docs.map(line => line.toLowerCase())
val words = lower.flatMap(line => line.split("s+"))
// Convert RDD to Dataframe using "Case class"
val words_df = words.map(Word(_)).toDF()
words_df.registerTempTable("words") // Register as a [TABLE]
val topWords = sqlContext.sql(
"SELECT text, count(text) AS n
FROM words
GROUP BY text
ORDER BY n DESC
LIMIT 10"
)
topWords.foreach(println)
5 . 3

case class Word(text: String)
val fileName = "README.md"
val docs = sc.textFile(fileName)
val lower = docs.map(line => line.toLowerCase())
val words = lower.flatMap(line => line.split("s+"))
val words_df = words.map(Word(_)).toDF()
words_df.registerTempTable("words")
val topWords = sqlContext.sql("SELECT text, count(text) AS n FROM words GROUP BY text ORDER BY
topWords.foreach(println)
5 . 19

import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.functions._
val sc = new SparkContext(conf)
// Spark SQL DataFrame, SQLContext
val sqlContext = new SQLContext(sc)

// RDD DataFrame
import sqlContext.implicits._
7 . 4

// Parquet DataFrame
val df = sqlContext.read.parquet("people.parquet")

// DataFrame stdout
// Displays the content of the DataFrame to stdout
df.show()
7 . 5

// DataFrame stdout
df.show()
// Schema
df.printSchema()
// "name"
df.select("name").show()
// "age" +1
df.select(df("name"), df("age") + 1).show()
// 21 people
df.filter(df("age") > 21).show()
// age count
df.groupBy("age").count().show()
7 . 6

//
df.registerTempTable("people")
// DataFrame stdout
sqlContext.sql("SELECT * FROM people").show()
// Schema
sqlContext.sql("SELECT * FROM people").printSchema()
// "name"
sqlContext.sql("SELECT name FROM people").show()
// "age" +1
sqlContext.sql("SELECT name, (age + 1) as age FROM people").show()
// 21 people
sqlContext.sql("SELECT * FROM people WHERE age > 21").show()
// age count
sqlContext.sql("SELECT age, count(age) as count FROM people Group By age").show()
7 . 7

// case class Schema
case class Person(name: String, age: Int)
// DataFrame
val df = sc.textFile("people.txt").map(_.split(",")).map(p => Person(p(0), p(1).trim.toInt)).t
df.registerTempTable("people")
val teenagers = sqlContext.sql("SELECT name, age FROM people WHERE age >= 13 AND age <= 19")
// SQL query DataFrame, normal RDD operation
teenagers.map( t => "Name: " + t(0)).collect().foreach(println)
//
teenagers.map (t => "Name: " + t.getAs[String]("name")).collect().foreach(println)
// row.getValueMap[T] Map[String, T]
teenagers.map (_.getValuesMap[Any](List("name", "age"))).collect().foreach(println)
7 . 8

// , "parquet" Spark
val df = sqlContext.read.load("users.parquet")
// DataFrame "parquet"
df.select("name", "favorite_color").write.save("namesAndFavColors.parquet")
7 . 9

val df = sqlContext.read.format("json").load("people.json")
// DataFrame "parquet"
df.select("name", "age").write.save("namesAndAges.parquet")
7 . 10

import org.apache.spark.sql.SaveMode
val df = sqlContext.read.load("users.parquet")

// DataFrame "parquet" ( SaveMode.Overwrite)
df.select("name", "favorite_color").write
.mode(SaveMode.Overwrite).save("namesAndFavColors.parquet")
7 . 11

//define the schema using a case class
case class Auction(auctionid: String, bid: Float, bidtime: Float, bidder: String, bidderrate:
// ebay auction
val ebayText = sc.textFile("ebay.csv")
// Auction
val ebay = ebayText.map(_.split(",")).map(p => Auction(p(0), p(1).toFloat, p(2).toFloat, p(3),
// DataFrame
val auction = ebay.toDF()
auction.registerTempTable("auction")
// ?
val count = auction.select("auctionid").distinct.count
System.out.println(count)

// (item)
val results =sqlContext.sql("SELECT auctionid, item, count(bid) as bid_count FROM auction GRO
results.show()

// ( / / ) 8 . 4

import com.databricks.spark.csv
// 3rd party library "CSV" Dataframe
val df = sqlContext.read
.format("com.databricks.spark.csv")
.option("header", "true") // Use first line of all files as header
.option("inferSchema", "true") // Automatically infer data types
.load("sfpd.csv")
// Schema
df.printSchema
// Distinct Category
df.select("Category").distinct().collect().foreach(println)
// temp table
df.registerTempTable("sfpd")
//
sqlContext.sql("SELECT distinct Category FROM sfpd").collect().foreach(println)
// Top 10
sqlContext.sql("SELECT Resolution , count(Resolution) as rescount FROM sfpd group by Resolutio
// Top 10
sqlContext.sql("SELECT Category , count(Category) as catcount FROM sfpd group by Category orde
8 . 6

Spark DataFrames and SQL

Recommended

Recommended

More Related Content

What's hot

What's hot (20)

Viewers also liked

Viewers also liked (20)

Similar to Spark DataFrames and SQL

Similar to Spark DataFrames and SQL (20)

More from Erhwen Kuo

More from Erhwen Kuo (19)

Recently uploaded

Recently uploaded (20)

Spark DataFrames and SQL