The document discusses Spark SQL's SQL parsing process. It explains that sqlContext.sql() uses a DDLParser and SparkSQLParser to parse the SQL string into a LogicalPlan using a ParserDialect. It then shows the classes involved in more detail, including the AbstractSparkSQLParser which is extended by SqlParser to define the parsing rules and parse the SQL into a LogicalPlan. It also mentions that Project Tungsten changed parts of the parsing from parser combinators to runtime code generation for performance.

1. SparkSQLの構文解析
dive into SparkSQL
株式会社サイバーエージェント 井上ゆり
@iyunoriue
http://x1.inkenkun.com/

2. sqlContext.sql("SELECT * FROM records")

3. sqlContext.sql("SELECT * FROM records")

4. def sql(sqlText: String): DataFrame = {
DataFrame(this, parseSql(sqlText))
}

5. 図示するとこんな感じ
DataFrame
sqlContext
parseSql:
LogicalPlan

6. parseSql: LogicalPlan
sqlText:String型 LogicalPlan型

7. protected[sql] def parseSql(sql: String): LogicalPlan =
ddlParser.parse(sql, false)
!
@transient
protected[sql] val ddlParser =
new DDLParser(sqlParser.parse(_))
!
@transient
protected[sql] val sqlParser =
new SparkSQLParser(getSQLDialect().parse(_))
!
!
protected[sql] def getSQLDialect(): ParserDialect = {
try {
val clazz = Utils.classForName(dialectClassName)
clazz.newInstance().asInstanceOf[ParserDialect]
} catch {
case NonFatal(e) =>
:
}
}

8. DDLParser
SparkSQLParser
ParserDialect
sql
⇒org.apache.spark.sql.catalyst.DefaultParserDialect
hiveql
⇒org.apache.spark.sql.hive.HiveQLDialect
方言、訛り
図示するとこんな感じ

9. LogicalPlan
LogicalPlan
LogicalPlan
DDLParser.parse
パース成功
パース失敗
SparkSQLParser.parse
パース成功
パース失敗
ParserDialect.parse

10. DDLParser
SparkSQLParser
!
} AbstractSparkSQLParser
のサブクラス
※ただし、DDLParserはAbstractSparkSQLParserのparseをオーバー
ライドしていますが、SparkSQLParserはAbstractSparkSQLParserの
parseをそのまま使っているため、parseの実装は異なります。

11. private[sql] abstract class AbstractSparkSQLParser
extends StandardTokenParsers with PackratParsers {
def parse(input: String): LogicalPlan = {
// Initialize the Keywords.
lexical.initialize(reservedWords)
phrase(start)(new lexical.Scanner(input)) match {
case Success(plan, _) => plan
case failureOrError =>
sys.error(failureOrError.toString)
}
}
:
}
今度はこの２つを
見てみます。

12. class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
protected val ABS = Keyword(“ABS")
protected val ALL = Keyword(“ALL”)
:
protected val WHEN = Keyword(“WHEN")
protected val WHERE = Keyword(“WHERE")
protected val WITH = Keyword("WITH")
!
protected lazy val start: Parser[LogicalPlan] =
start1 | insert | cte
!
protected lazy val start1: Parser[LogicalPlan] =
(select | ("(" ~> select <~ ")")) *
( UNION ~ ALL ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Union(q1, q2) }
| INTERSECT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Intersect(q1, q2) }
| EXCEPT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Except(q1, q2)}
| UNION ~ DISTINCT.? ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Distinct(Union(q1, q2)) }
)
!
protected lazy val select: Parser[LogicalPlan] =
SELECT ~> DISTINCT.? ~
repsep(projection, ",") ~
(FROM ~> relations).? ~
(WHERE ~> expression).? ~
(GROUP ~ BY ~> rep1sep(expression, ",")).? ~
(HAVING ~> expression).? ~
sortType.? ~
(LIMIT ~> expression).? ^^ {
case d ~ p ~ r ~ f ~ g ~ h ~ o ~ l =>
val base = r.getOrElse(OneRowRelation)
val withFilter = f.map(Filter(_, base)).getOrElse(base)
val withProjection = g
.map(Aggregate(_, assignAliases(p), withFilter))
.getOrElse(Project(assignAliases(p), withFilter))
val withDistinct = d.map(_ => Distinct(withProjection)).getOrElse(withProjection)
val withHaving = h.map(Filter(_, withDistinct)).getOrElse(withDistinct)
val withOrder = o.map(_(withHaving)).getOrElse(withHaving)
val withLimit = l.map(Limit(_, withOrder)).getOrElse(withOrder)
withLimit
}
!
protected lazy val insert: Parser[LogicalPlan] =
INSERT ~> (OVERWRITE ^^^ true | INTO ^^^ false) ~ (TABLE ~> relation) ~ select ^^ {
case o ~ r ~ s => InsertIntoTable(r, Map.empty[String, Option[String]], s, o, false)
}
:
}

13. class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
protected val ABS = Keyword(“ABS")
protected val ALL = Keyword(“ALL”)
:
protected val WHEN = Keyword(“WHEN")
protected val WHERE = Keyword(“WHERE")
protected val WITH = Keyword("WITH")
!
protected lazy val start: Parser[LogicalPlan] =
start1 | insert | cte
!
protected lazy val start1: Parser[LogicalPlan] =
(select | ("(" ~> select <~ ")")) *
( UNION ~ ALL ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Union(q1, q2) }
| INTERSECT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Intersect(q1, q2) }
| EXCEPT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Except(q1, q2)}
| UNION ~ DISTINCT.? ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Distinct(Union(q1, q2)) }
)
!
protected lazy val select: Parser[LogicalPlan] =
SELECT ~> DISTINCT.? ~
repsep(projection, ",") ~
(FROM ~> relations).? ~
(WHERE ~> expression).? ~
(GROUP ~ BY ~> rep1sep(expression, ",")).? ~
(HAVING ~> expression).? ~
sortType.? ~
(LIMIT ~> expression).? ^^ {
case d ~ p ~ r ~ f ~ g ~ h ~ o ~ l =>
val base = r.getOrElse(OneRowRelation)
val withFilter = f.map(Filter(_, base)).getOrElse(base)
val withProjection = g
.map(Aggregate(_, assignAliases(p), withFilter))
.getOrElse(Project(assignAliases(p), withFilter))
val withDistinct = d.map(_ => Distinct(withProjection)).getOrElse(withProjection)
val withHaving = h.map(Filter(_, withDistinct)).getOrElse(withDistinct)
val withOrder = o.map(_(withHaving)).getOrElse(withHaving)
val withLimit = l.map(Limit(_, withOrder)).getOrElse(withOrder)
withLimit
}
!
protected lazy val insert: Parser[LogicalPlan] =
INSERT ~> (OVERWRITE ^^^ true | INTO ^^^ false) ~ (TABLE ~> relation) ~ select ^^ {
case o ~ r ~ s => InsertIntoTable(r, Map.empty[String, Option[String]], s, o, false)
}
:
}
そうです。おなじみの
パーサー・コンビネータです。

14. Project Tungsten
クエリ構文解析の一部をパーサー・コンビネータ
からRuntime Code Generationに変更。
なぜ？
パーサー・コンビネータは決して早くない
(計算量:O(n^k))
quasiquotesを使ってruntimeにコードを生成
Deep Dive into Spark SQL’s Catalyst Optimizer:
https://databricks.com/blog/2015/04/13/deep-dive-into-spark-sqls-catalyst-optimizer.html

15. def compile(node: Node): AST = node match {
case Literal(value) => q"$value"
case Attribute(name) => q"row.get($name)"
case Add(left, right) => q"${compile(left)} + ${compile(right)}"
}