[SPARK-2540] [SQL] Add HiveDecimal & HiveVarchar support in unwrapping data
authorCheng Hao <hao.cheng@intel.com>
Fri, 18 Jul 2014 21:38:11 +0000 (16:38 -0500)
committerMichael Armbrust <michael@databricks.com>
Fri, 18 Jul 2014 21:38:32 +0000 (16:38 -0500)
Author: Cheng Hao <hao.cheng@intel.com>

Closes #1436 from chenghao-intel/unwrapdata and squashes the following commits:

34cc21a [Cheng Hao] update the table scan accodringly since the unwrapData function changed
afc39da [Cheng Hao] Polish the code
39d6475 [Cheng Hao] Add HiveDecimal & HiveVarchar support in unwrap data

(cherry picked from commit 7f1720813793e155743b58eae5228298e894b90d)
Signed-off-by: Michael Armbrust <michael@databricks.com>
sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala
sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala

index ef8bae7..e7016fa 100644 (file)
@@ -96,19 +96,9 @@ case class HiveTableScan(
           .getOrElse(sys.error(s"Can't find attribute $a"))
         val fieldObjectInspector = ref.getFieldObjectInspector
 
-        val unwrapHiveData = fieldObjectInspector match {
-          case _: HiveVarcharObjectInspector =>
-            (value: Any) => value.asInstanceOf[HiveVarchar].getValue
-          case _: HiveDecimalObjectInspector =>
-            (value: Any) => BigDecimal(value.asInstanceOf[HiveDecimal].bigDecimalValue())
-          case _ =>
-            identity[Any] _
-        }
-
         (row: Any, _: Array[String]) => {
           val data = objectInspector.getStructFieldData(row, ref)
-          val hiveData = unwrapData(data, fieldObjectInspector)
-          if (hiveData != null) unwrapHiveData(hiveData) else null
+          unwrapData(data, fieldObjectInspector)
         }
       }
     }
index 9b10530..fc33c5b 100644 (file)
@@ -280,6 +280,10 @@ private[hive] case class HiveGenericUdf(name: String, children: Seq[Expression])
 private[hive] trait HiveInspectors {
 
   def unwrapData(data: Any, oi: ObjectInspector): Any = oi match {
+    case hvoi: HiveVarcharObjectInspector => 
+      if (data == null) null else hvoi.getPrimitiveJavaObject(data).getValue
+    case hdoi: HiveDecimalObjectInspector => 
+      if (data == null) null else BigDecimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue())
     case pi: PrimitiveObjectInspector => pi.getPrimitiveJavaObject(data)
     case li: ListObjectInspector =>
       Option(li.getList(data))