批量导入数据到hive表中：假设我有60张主子表如何批量创建导入数据

背景：根据业务需要需要把60张主子表批量入库到hive表。

创建测试数据：

  def createBatchTestFile(): Unit = {
    for (layer <- 0 to 59) {
      val sWriter = new PrintWriter(new File("D:\\server_" + layer + ".txt"))
      val nWriter = new PrintWriter(new File("D:\\neighour_" + layer + ".txt"))

      sWriter.write(layer + "," + "server_" + layer)
      nWriter.write(layer + "," + "neighour_" + layer)

      sWriter.close()
      nWriter.close()
    }
  }

上传数据到服务器hdfs: 

[hadoop@vd01 fglib]$ ls
neighour_0.txt   neighour_20.txt  neighour_31.txt  neighour_42.txt  neighour_53.txt  server_0.txt   server_20.txt  server_31.txt  server_42.txt  server_53.txt
neighour_10.txt  neighour_21.txt  neighour_32.txt  neighour_43.txt  neighour_54.txt  server_10.txt  server_21.txt  server_32.txt  server_43.txt  server_54.txt
neighour_11.txt  neighour_22.txt  neighour_33.txt  neighour_44.txt  neighour_55.txt  server_11.txt  server_22.txt  server_33.txt  server_44.txt  server_55.txt
neighour_12.txt  neighour_23.txt  neighour_34.txt  neighour_45.txt  neighour_56.txt  server_12.txt  server_23.txt  server_34.txt  server_45.txt  server_56.txt
neighour_13.txt  neighour_24.txt  neighour_35.txt  neighour_46.txt  neighour_57.txt  server_13.txt  server_24.txt  server_35.txt  server_46.txt  server_57.txt
neighour_14.txt  neighour_25.txt  neighour_36.txt  neighour_47.txt  neighour_58.txt  server_14.txt  server_25.txt  server_36.txt  server_47.txt  server_58.txt
neighour_15.txt  neighour_26.txt  neighour_37.txt  neighour_48.txt  neighour_59.txt  server_15.txt  server_26.txt  server_37.txt  server_48.txt  server_59.txt
neighour_16.txt  neighour_27.txt  neighour_38.txt  neighour_49.txt  neighour_5.txt   server_16.txt  server_27.txt  server_38.txt  server_49.txt  server_5.txt
neighour_17.txt  neighour_28.txt  neighour_39.txt  neighour_4.txt   neighour_6.txt   server_17.txt  server_28.txt  server_39.txt  server_4.txt   server_6.txt
neighour_18.txt  neighour_29.txt  neighour_3.txt   neighour_50.txt  neighour_7.txt   server_18.txt  server_29.txt  server_3.txt   server_50.txt  server_7.txt
neighour_19.txt  neighour_2.txt   neighour_40.txt  neighour_51.txt  neighour_8.txt   server_19.txt  server_2.txt   server_40.txt  server_51.txt  server_8.txt
neighour_1.txt   neighour_30.txt  neighour_41.txt  neighour_52.txt  neighour_9.txt   server_1.txt   server_30.txt  server_41.txt  server_52.txt  server_9.txt
[hadoop@vd01 fglib]$ hadoop fs -mkdir /user/hive_user/dang/fglib
17/09/07 13:24:14 INFO hdfs.PeerCache: SocketCache disabled.
[hadoop@vd01 fglib]$ hadoop fs -copyFromLocal * /user/hive_user/dang/fglib/
17/09/07 13:24:39 INFO hdfs.PeerCache: SocketCache disabled.
[hadoop@vd01 fglib]$ 

[hadoop@vd01 fglib]$ hadoop fs -ls /user/hive_user/dang/fglib
17/09/07 13:25:51 INFO hdfs.PeerCache: SocketCache disabled.
Found 120 items
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_0.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_1.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_10.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_11.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_12.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_13.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_14.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_15.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_16.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_17.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_18.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_19.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_2.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_20.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_21.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_22.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_23.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_24.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_25.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_26.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_27.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_28.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_29.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_3.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_30.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_31.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_32.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_33.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_34.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_35.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_36.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_37.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_38.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_39.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_4.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_40.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_41.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_42.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_43.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_44.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_45.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_46.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_47.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_48.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_49.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_5.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_50.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_51.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_52.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_53.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_54.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_55.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_56.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_57.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_58.txt
-rw-r--r--+  3 hive_user hadoop         14 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_59.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_6.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_7.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_8.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/neighour_9.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:24 /user/hive_user/dang/fglib/server_0.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:24 /user/hive_user/dang/fglib/server_1.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_10.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_11.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_12.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_13.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_14.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_15.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_16.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_17.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_18.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_19.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:24 /user/hive_user/dang/fglib/server_2.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_20.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_21.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_22.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_23.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_24.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_25.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_26.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_27.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_28.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_29.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:25 /user/hive_user/dang/fglib/server_3.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_30.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_31.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_32.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:24 /user/hive_user/dang/fglib/server_33.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_34.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_35.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_36.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_37.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_38.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_39.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:25 /user/hive_user/dang/fglib/server_4.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_40.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_41.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_42.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_43.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_44.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_45.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_46.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_47.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_48.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_49.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:25 /user/hive_user/dang/fglib/server_5.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_50.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_51.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_52.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_53.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_54.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_55.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_56.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_57.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_58.txt
-rw-r--r--+  3 hive_user hadoop         12 2017-09-07 13:25 /user/hive_user/dang/fglib/server_59.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:25 /user/hive_user/dang/fglib/server_6.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:25 /user/hive_user/dang/fglib/server_7.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:25 /user/hive_user/dang/fglib/server_8.txt
-rw-r--r--+  3 hive_user hadoop         10 2017-09-07 13:25 /user/hive_user/dang/fglib/server_9.txt

实现批量创建表及批量入库：

val conf = new SparkConf().setAppName("My_Test")
    //.setMaster("local[1]").setMaster("spark://xx.xx.xx.xx:7077").setJars(List("xxx.jar")).set("spark.executor.memory", "10g")
    val sc = new SparkContext(conf)
    val hiveContext = new HiveContext(sc)
    // use my_hive_db;
    hiveContext.sql("use my_hive_db")

    import hiveContext.implicits._

    hiveContext.setConf("mapred.max.split.size", "256000000")
    hiveContext.setConf("mapred.min.split.size.per.node", "100000000")
    hiveContext.setConf("mapred.min.split.size.per.rack", "100000000")
    hiveContext.setConf("hive.input.format", "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat")
    hiveContext.setConf("hive.merge.mapfiles", "true")
    hiveContext.setConf("hive.merge.mapredfiles", "true")
    hiveContext.setConf("hive.merge.size.per.task", "256000000")
    hiveContext.setConf("hive.merge.smallfiles.avgsize", "256000000")
    hiveContext.setConf("hive.groupby.skewindata", "true")

//    for (layer <- 0 to 59) {
//      hiveContext.sql("create table if not exists bathServer_" + layer + "(id int,name string) row format delimited fields terminated by ',' stored as textfile")
//      hiveContext.sql("create table if not exists bathNeighbour_" + layer + "(id int,name string) row format delimited fields terminated by ',' stored as textfile")
//      hiveContext.sql("load data inpath 'hdfs:/user/userxx/dang/server_" + layer + ".txt' into table bathServer_"+ layer )
//      hiveContext.sql("load data inpath 'hdfs:/user/userxx/dang/neighour_" + layer + ".txt' into table bathNeighbour_"+ layer )
//    }

    sc.parallelize(0 to 59).collect().foreach((layer)=>{
        hiveContext.sql("create table if not exists bathServer_" + layer + "(id int,name string) row format delimited fields terminated by ',' stored as textfile")
        hiveContext.sql("create table if not exists bathNeighbour_" + layer + "(id int,name string) row format delimited fields terminated by ',' stored as textfile")
        hiveContext.sql("load data inpath 'hdfs:/user/userxx/dang/fglib/server_" + layer + ".txt' into table bathServer_"+ layer )
        hiveContext.sql("load data inpath 'hdfs:/user/userxx/dang/fglib/neighour_" + layer + ".txt' into table bathNeighbour_"+ layer )
    })

    sc.stop()
  }