使用 Scala 将 SparkRDD 写入 HBase table

Question

我正在尝试使用 scala（之前未使用过）将 SparkRDD 写入 HBase table。整个代码是这样的：

import org.apache.hadoop.hbase.client.{HBaseAdmin, Result}
import org.apache.hadoop.hbase.{HBaseConfiguration, HTableDescriptor}
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.io.ImmutableBytesWritable    
import scala.collection.JavaConverters._
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark._
import org.apache.hadoop.mapred.JobConf
import org.apache.spark.rdd.PairRDDFunctions
import org.apache.spark.SparkContext._
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.client._

object HBaseWrite {
   def main(args: Array[String]) {
     val sparkConf = new SparkConf().setAppName("HBaseWrite").setMaster("local").set("spark.driver.allowMultipleContexts","true").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
     val sc = new SparkContext(sparkConf)
     val conf = HBaseConfiguration.create()
     val outputTable = "tablename"

     System.setProperty("user.name", "hdfs")
     System.setProperty("HADOOP_USER_NAME", "hdfs")
     conf.set("hbase.master", "localhost:60000")
     conf.setInt("timeout", 120000)
     conf.set("hbase.zookeeper.quorum", "localhost")
     conf.set("zookeeper.znode.parent", "/hbase-unsecure")
     conf.setInt("hbase.client.scanner.caching", 10000)
     sparkConf.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.client.Result]))
     val jobConfig: JobConf = new JobConf(conf,this.getClass)
     jobConfig.setOutputFormat(classOf[TableOutputFormat])
     jobConfig.set(TableOutputFormat.OUTPUT_TABLE,outputTable)
     val x = 12
     val y = 15
     val z = 25
     var newarray = Array(x,y,z)
     val newrddtohbase = sc.parallelize(newarray)
     def convert(a:Int) : Tuple2[ImmutableBytesWritable,Put] = {
          val p = new Put(Bytes.toBytes(a))
          p.add(Bytes.toBytes("columnfamily"),
          Bytes.toBytes("col_1"), Bytes.toBytes(a))
          new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
     }
     new PairRDDFunctions(newrddtohbase.map(convert)).saveAsHadoopDataset(jobConfig)
     sc.stop()
   }
}

我在执行 HBaseWrite(main(Array()) 后得到的错误是这样的：

org.apache.spark.SparkException: Task not serializable

我该如何继续完成它？

Answer 1

你在这里做错的是在 main 中定义了 convert 如果您以这种方式编写此代码，它可能会起作用：

    object HBaseWrite {
       def main(args: Array[String]) {
         val sparkConf = new SparkConf().setAppName("HBaseWrite").setMaster("local").set("spark.driver.allowMultipleContexts","true").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
         val sc = new SparkContext(sparkConf)
         val conf = HBaseConfiguration.create()
         val outputTable = "tablename"

         System.setProperty("user.name", "hdfs")
         System.setProperty("HADOOP_USER_NAME", "hdfs")
         conf.set("hbase.master", "localhost:60000")
         conf.setInt("timeout", 120000)
         conf.set("hbase.zookeeper.quorum", "localhost")
         conf.set("zookeeper.znode.parent", "/hbase-unsecure")
         conf.setInt("hbase.client.scanner.caching", 10000)
         sparkConf.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.client.Result]))
         val jobConfig: JobConf = new JobConf(conf,this.getClass)
         jobConfig.setOutputFormat(classOf[TableOutputFormat])
         jobConfig.set(TableOutputFormat.OUTPUT_TABLE,outputTable)
         val x = 12
         val y = 15
         val z = 25
         var newarray = Array(x,y,z)
         val newrddtohbase = sc.parallelize(newarray)
         val convertFunc = convert _
         new PairRDDFunctions(newrddtohbase.map(convertFunc)).saveAsHadoopDataset(jobConfig)
         sc.stop()
       }
       def convert(a:Int) : Tuple2[ImmutableBytesWritable,Put] = {
              val p = new Put(Bytes.toBytes(a))
              p.add(Bytes.toBytes("columnfamily"),
              Bytes.toBytes("col_1"), Bytes.toBytes(a))
              new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
         }
    }

P.S.: 代码未经测试，但应该可以工作！

Answer 2

例如，下面的方法将 Int 作为参数并且 returns Double

var toDouble: (Int) => Double = a => {
    a.toDouble
}

您可以使用 toDouble(2) 并且它 returns 2.0

您可以按照以下相同的方式将方法转换为函数文字。

val convert: (Int) => Tuple2[ImmutableBytesWritable,Put] = a => {
              val p = new Put(Bytes.toBytes(a))
              p.add(Bytes.toBytes("columnfamily"),
              Bytes.toBytes("col_1"), Bytes.toBytes(a))
              new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
         }

使用 Scala 将 SparkRDD 写入 HBase table

writing SparkRDD to a HBase table using Scala

hbase

scala

apache-spark

rdd