categorys
//spark-shell --driver-class-path /home/hadoop/test/mysqljdbc.jar
import java.sql.DriverManager
// --queue szbigdata
import org.apache.spark.{SparkConf, SparkContext}
object categorys {
val rl= "jdbc:mysql://10.19.65.17:54321/emotion?user=emotion&password=qingxu"
//jdbc:mysql://10.19.65.17:54322/emotion?user=emotion&password=qingxu
classOf[com.mysql.jdbc.Driver]
val conn = DriverManager.getConnection(rl)
def main(args: Array[String]) {
val conf = new SparkConf().setAppName("categorys")
val sc = new SparkContext(conf)
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
val role = "jdbc:mysql://192.168.0.37:3306/emotional?user=root&password=123456&useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false"
val log881101 = sc.textFile("hdfs://ns1/user/songchunlin/881101.种植业与林业.txt")
val format = new java.text.SimpleDateFormat("yyyyMMdd")
val date = format.format(new java.util.Date().getTime()).toString
val rowRDD881101= log881101.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
import sqlContext.implicits._
val slog881101 = rowRDD881101.map(line => (date, "881101", "种植业与林业", line._4)).toDF("editdate", "number", "category", "words")
val log881102 = sc.textFile("hdfs://ns1/user/songchunlin/881102.养殖业.txt")
val rowRDDlog881102=log881102.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881102=rowRDDlog881102.map(line=>(date,"881102","养殖业",line._4)).toDF("editdate", "number", "category", "words")
val log881103 = sc.textFile("hdfs://ns1/user/songchunlin/881103.农产品加工.txt")
val rowRDDlog881103=log881103.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881103=rowRDDlog881103.map(line=>(date,"881103","农产品加工",line._4)).toDF("editdate", "number", "category", "words")
val log881104 = sc.textFile("hdfs://ns1/user/songchunlin/881104.农业服务.txt")
val rowRDDlog881104=log881104.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881104=rowRDDlog881104.map(line=>(date,"881104","农业服务",line._4)).toDF("editdate", "number", "category", "words")
val log881105 = sc.textFile("hdfs://ns1/user/songchunlin/881105.煤炭开采.txt")
val rowRDDlog881105=log881105.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881105=rowRDDlog881105.map(line=>(date,"881105","煤炭开采",line._4)).toDF("editdate", "number", "category", "words")
val log881106 = sc.textFile("hdfs://ns1/user/songchunlin/881106.石油矿业开采.txt")
val rowRDDlog881106=log881106.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881106=rowRDDlog881106.map(line=>(date,"881106","石油矿业开采",line._4)).toDF("editdate", "number", "category", "words")
val log881107 = sc.textFile("hdfs://ns1/user/songchunlin/881107.采掘服务.txt")
val rowRDDlog881107=log881107.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881107=rowRDDlog881107.map(line=>(date,"881107","采掘服务",line._4)).toDF("editdate", "number", "category", "words")
val log881108 = sc.textFile("hdfs://ns1/user/songchunlin/881108.基础化学.txt")
val rowRDDlog881108=log881108.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881108=rowRDDlog881108.map(line=>(date,"881108","基础化学",line._4)).toDF("editdate", "number", "category", "words")
val log881109 = sc.textFile("hdfs://ns1/user/songchunlin/881109.化学制品.txt")
val rowRDDlog881109=log881109.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881109=rowRDDlog881109.map(line=>(date,"881109","化学制品",line._4)).toDF("editdate", "number", "category", "words")
val log881110 = sc.textFile("hdfs://ns1/user/songchunlin/881110.化工合成材料.txt")
val rowRDDlog881110=log881110.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881110=rowRDDlog881110.map(line=>(date,"881110","化工合成材料",line._4)).toDF("editdate", "number", "category", "words")
val log881111 = sc.textFile("hdfs://ns1/user/songchunlin/881111.化工新材料.txt")
val rowRDDlog881111=log881111.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881111=rowRDDlog881111.map(line=>(date,"881111","化工新材料",line._4)).toDF("editdate", "number", "category", "words")
val log881112 = sc.textFile("hdfs://ns1/user/songchunlin/881112.钢铁.txt")
val rowRDDlog881112=log881112.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881112=rowRDDlog881112.map(line=>(date,"881112","钢铁",line._4)).toDF("editdate", "number", "category", "words")
val log881113 = sc.textFile("hdfs://ns1/user/songchunlin/881113.有色冶炼加工.txt")
val rowRDDlog881113=log881113.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881113=rowRDDlog881113.map(line=>(date,"881113","有色冶炼加工",line._4)).toDF("editdate", "number", "category", "words")
val log881114 = sc.textFile("hdfs://ns1/user/songchunlin/881114.新材料.txt")
val rowRDDlog881114=log881114.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881114=rowRDDlog881114.map(line=>(date,"881114","新材料",line._4)).toDF("editdate", "number", "category", "words")
val log881115 = sc.textFile("hdfs://ns1/user/songchunlin/881115.建筑材料.txt")
val rowRDDlog881115=log881115.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881115=rowRDDlog881115.map(line=>(date,"881115","建筑材料",line._4)).toDF("editdate", "number", "category", "words")
val log881116 = sc.textFile("hdfs://ns1/user/songchunlin/881116.建筑装饰.txt")
val rowRDDlog881116=log881116.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881116=rowRDDlog881116.map(line=>(date,"881116","建筑材料",line._4)).toDF("editdate", "number", "category", "words")
val log881117 = sc.textFile("hdfs://ns1/user/songchunlin/881117.通用设备.txt")
val rowRDDlog881117=log881117.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881117=rowRDDlog881117.map(line=>(date,"881117","通用设备",line._4)).toDF("editdate", "number", "category", "words")
val log881118 = sc.textFile("hdfs://ns1/user/songchunlin/881118.专用设备.txt")
val rowRDDlog881118=log881118.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881118=rowRDDlog881118.map(line=>(date,"881118","专用设备",line._4)).toDF("editdate", "number", "category", "words")
val log881119 = sc.textFile("hdfs://ns1/user/songchunlin/881119.仪器仪表.txt")
val rowRDDlog881119=log881119.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881119=rowRDDlog881119.map(line=>(date,"881119","仪器仪表",line._4)).toDF("editdate", "number", "category", "words")
val log881120 = sc.textFile("hdfs://ns1/user/songchunlin/881120.电气设备.txt")
val rowRDDlog881120=log881120.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881120=rowRDDlog881120.map(line=>(date,"881120","电气设备",line._4)).toDF("editdate", "number", "category", "words")
val log881121 = sc.textFile("hdfs://ns1/user/songchunlin/881121.半导体及元件.txt")
val rowRDDlog881121=log881121.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881121=rowRDDlog881121.map(line=>(date,"881121","半导体及元件",line._4)).toDF("editdate", "number", "category", "words")
val log881122 = sc.textFile("hdfs://ns1/user/songchunlin/881122.光学光电子.txt")
val rowRDDlog881122=log881122.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881122=rowRDDlog881122.map(line=>(date,"881122","光学光电子",line._4)).toDF("editdate", "number", "category", "words")
val log881123 = sc.textFile("hdfs://ns1/user/songchunlin/881123.其他电子.txt")
val rowRDDlog881123=log881123.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881123=rowRDDlog881123.map(line=>(date,"881123","其他电子",line._4)).toDF("editdate", "number", "category", "words")
val log881124 = sc.textFile("hdfs://ns1/user/songchunlin/881124.电子制造.txt")
val rowRDDlog881124=log881124.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881124=rowRDDlog881124.map(line=>(date,"881124","电子制造",line._4)).toDF("editdate", "number", "category", "words")
val log881125 = sc.textFile("hdfs://ns1/user/songchunlin/881125.汽车整车.txt")
val rowRDDlog881125=log881125.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881125=rowRDDlog881125.map(line=>(date,"881125","汽车整车",line._4)).toDF("editdate", "number", "category", "words")
val log881126 = sc.textFile("hdfs://ns1/user/songchunlin/881126.汽车零部件.txt")
val rowRDDlog881126=log881126.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881126=rowRDDlog881126.map(line=>(date,"881126","汽车零部件",line._4)).toDF("editdate", "number", "category", "words")
val log881127 = sc.textFile("hdfs://ns1/user/songchunlin/881127.非汽车交运.txt")
val rowRDDlog881127=log881127.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881127=rowRDDlog881127.map(line=>(date,"881127","非汽车交运",line._4)).toDF("editdate", "number", "category", "words")
val log881128 = sc.textFile("hdfs://ns1/user/songchunlin/881128.交运设备服务.txt")
val rowRDDlog881128=log881128.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881128=rowRDDlog881128.map(line=>(date,"881128","交运设备服务",line._4)).toDF("editdate", "number", "category", "words")
val log881129 = sc.textFile("hdfs://ns1/user/songchunlin/881129.通信设备.txt")
val rowRDDlog881129=log881129.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881129=rowRDDlog881129.map(line=>(date,"881129","通信设备",line._4)).toDF("editdate", "number", "category", "words")
val log881130 = sc.textFile("hdfs://ns1/user/songchunlin/881130.计算机设备.txt")
val rowRDDlog881130=log881130.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
val slog881130=rowRDDlog881130.map(line=>(date,"881130","计算机设备",line._4)).toDF("editdate", "number", "category", "words")
val log881131 = sc.textFile("hdfs://ns1/user/songchunlin/881131.白色家电.txt")
val rowRDDlog881131 = log881131.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881131 = rowRDDlog881131.map(line => (date, "881131", "白色家电", line._4)).toDF("editdate", "number", "category", "words")
val log881132 = sc.textFile("hdfs://ns1/user/songchunlin/881132.视听器材.txt")
val rowRDDlog881132 = log881132.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881132 = rowRDDlog881132.map(line => (date, "881132", "视听器材", line._4)).toDF("editdate", "number", "category", "words")
val log881133 = sc.textFile("hdfs://ns1/user/songchunlin/881133.饮料制造.txt")
val rowRDDlog881133 = log881133.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881133 = rowRDDlog881133.map(line => (date, "881133", "饮料制造", line._4)).toDF("editdate", "number", "category", "words")
val log881134 = sc.textFile("hdfs://ns1/user/songchunlin/881134.食品加工制造.txt")
val rowRDDlog881134 = log881134.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881134 = rowRDDlog881134.map(line => (date, "881134", "食品加工制造", line._4)).toDF("editdate", "number", "category", "words")
val log881135 = sc.textFile("hdfs://ns1/user/songchunlin/881135.纺织制造.txt")
val rowRDDlog881135 = log881135.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881135 = rowRDDlog881135.map(line => (date, "881135", "纺织制造", line._4)).toDF("editdate", "number", "category", "words")
val log881136 = sc.textFile("hdfs://ns1/user/songchunlin/881136.服装家纺.txt")
val rowRDDlog881136 = log881136.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881136 = rowRDDlog881136.map(line => (date, "881136", "服装家纺", line._4)).toDF("editdate", "number", "category", "words")
val log881137 = sc.textFile("hdfs://ns1/user/songchunlin/881137.造纸.txt")
val rowRDDlog881137 = log881137.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881137 = rowRDDlog881137.map(line => (date, "881137", "造纸", line._4)).toDF("editdate", "number", "category", "words")
val log881138 = sc.textFile("hdfs://ns1/user/songchunlin/881138.包装印刷.txt")
val rowRDDlog881138 = log881138.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881138 = rowRDDlog881138.map(line => (date, "881138", "包装印刷", line._4)).toDF("editdate", "number", "category", "words")
val log881139 = sc.textFile("hdfs://ns1/user/songchunlin/881139.家用轻工.txt")
val rowRDDlog881139 = log881139.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881139 = rowRDDlog881139.map(line => (date, "881139", "家用轻工", line._4)).toDF("editdate", "number", "category", "words")
val log881140 = sc.textFile("hdfs://ns1/user/songchunlin/881140.化学制药.txt")
val rowRDDlog881140 = log881140.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881140 = rowRDDlog881140.map(line => (date, "881140", "化学制药", line._4)).toDF("editdate", "number", "category", "words")
val log881141 = sc.textFile("hdfs://ns1/user/songchunlin/881141.中药.txt")
val rowRDDlog881141 = log881141.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881141 = rowRDDlog881141.map(line => (date, "881141", "中药", line._4)).toDF("editdate", "number", "category", "words")
val log881142 = sc.textFile("hdfs://ns1/user/songchunlin/881142.生物制品.txt")
val rowRDDlog881142 = log881142.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881142 = rowRDDlog881142.map(line => (date, "881142", "生物制品", line._4)).toDF("editdate", "number", "category", "words")
val log881143 = sc.textFile("hdfs://ns1/user/songchunlin/881143.医药商业.txt")
val rowRDDlog881143 = log881143.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881143 = rowRDDlog881143.map(line => (date, "881143", "医药商业", line._4)).toDF("editdate", "number", "category", "words")
val log881144 = sc.textFile("hdfs://ns1/user/songchunlin/881144.医疗器械服务.txt")
val rowRDDlog881144 = log881144.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881144 = rowRDDlog881144.map(line => (date, "881144", "医疗器械服务", line._4)).toDF("editdate", "number", "category", "words")
val log881145 = sc.textFile("hdfs://ns1/user/songchunlin/881145.电力.txt")
val rowRDDlog881145 = log881145.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881145 = rowRDDlog881145.map(line => (date, "881145", "电力", line._4)).toDF("editdate", "number", "category", "words")
val log881146 = sc.textFile("hdfs://ns1/user/songchunlin/881146.燃气水务.txt")
val rowRDDlog881146 = log881146.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881146 = rowRDDlog881146.map(line => (date, "881146", "燃气水务", line._4)).toDF("editdate", "number", "category", "words")
val log881147 = sc.textFile("hdfs://ns1/user/songchunlin/881147.环保工程.txt")
val rowRDDlog881147 = log881147.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881147 = rowRDDlog881147.map(line => (date, "881147", "环保工程", line._4)).toDF("editdate", "number", "category", "words")
val log881148 = sc.textFile("hdfs://ns1/user/songchunlin/881148.港口航运.txt")
val rowRDDlog881148 = log881148.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881148 = rowRDDlog881148.map(line => (date, "881148", "港口航运", line._4)).toDF("editdate", "number", "category", "words")
val log881149 = sc.textFile("hdfs://ns1/user/songchunlin/881149.公路铁路运输.txt")
val rowRDDlog881149 = log881149.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881149 = rowRDDlog881149.map(line => (date, "881149", "公路铁路运输", line._4)).toDF("editdate", "number", "category", "words")
val log881150 = sc.textFile("hdfs://ns1/user/songchunlin/881150.公交.txt")
val rowRDDlog881150 = log881150.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881150 = rowRDDlog881150.map(line => (date, "881150", "公交", line._4)).toDF("editdate", "number", "category", "words")
val log881151 = sc.textFile("hdfs://ns1/user/songchunlin/881151.机场航运.txt")
val rowRDDlog881151 = log881151.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881151 = rowRDDlog881151.map(line => (date, "881151", "机场航运", line._4)).toDF("editdate", "number", "category", "words")
val log881152 = sc.textFile("hdfs://ns1/user/songchunlin/881152.物流.txt")
val rowRDDlog881152 = log881152.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881152 = rowRDDlog881152.map(line => (date, "881152", "物流", line._4)).toDF("editdate", "number", "category", "words")
val log881153 = sc.textFile("hdfs://ns1/user/songchunlin/881153.房地产开发.txt")
val rowRDDlog881153 = log881153.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881153 = rowRDDlog881153.map(line => (date, "881153", "房地产开发", line._4)).toDF("editdate", "number", "category", "words")
val log881154 = sc.textFile("hdfs://ns1/user/songchunlin/881154.园区开发.txt")
val rowRDDlog881154 = log881154.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881154 = rowRDDlog881154.map(line => (date, "881154", "园区开发", line._4)).toDF("editdate", "number", "category", "words")
val log881155 = sc.textFile("hdfs://ns1/user/songchunlin/881155.银行.txt")
val rowRDDlog881155 = log881155.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881155 = rowRDDlog881155.map(line => (date, "881155", "银行", line._4)).toDF("editdate", "number", "category", "words")
val log881156 = sc.textFile("hdfs://ns1/user/songchunlin/881156.保险及其他.txt")
val rowRDDlog881156 = log881156.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881156 = rowRDDlog881156.map(line => (date, "881156", "保险及其他", line._4)).toDF("editdate", "number", "category", "words")
val log881157 = sc.textFile("hdfs://ns1/user/songchunlin/881157.证券.txt")
val rowRDDlog881157 = log881157.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881157 = rowRDDlog881157.map(line => (date, "881157", "证券", line._4)).toDF("editdate", "number", "category", "words")
val log881158 = sc.textFile("hdfs://ns1/user/songchunlin/881158.零售.txt")
val rowRDDlog881158 = log881158.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881158 = rowRDDlog881158.map(line => (date, "881158", "零售", line._4)).toDF("editdate", "number", "category", "words")
val log881159 = sc.textFile("hdfs://ns1/user/songchunlin/881159.贸易.txt")
val rowRDDlog881159 = log881159.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881159 = rowRDDlog881159.map(line => (date, "881159", "贸易", line._4)).toDF("editdate", "number", "category", "words")
val log881160 = sc.textFile("hdfs://ns1/user/songchunlin/881160.景点及旅游.txt")
val rowRDDlog881160 = log881160.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881160 = rowRDDlog881160.map(line => (date, "881160", "景点及旅游", line._4)).toDF("editdate", "number", "category", "words")
val log881161 = sc.textFile("hdfs://ns1/user/songchunlin/881161.酒店及餐饮.txt")
val rowRDDlog881161 = log881161.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881161 = rowRDDlog881161.map(line => (date, "881161", "酒店及餐饮", line._4)).toDF("editdate", "number", "category", "words")
val log881162 = sc.textFile("hdfs://ns1/user/songchunlin/881162.通信服务.txt")
val rowRDDlog881162 = log881162.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881162 = rowRDDlog881162.map(line => (date, "881162", "通信服务", line._4)).toDF("editdate", "number", "category", "words")
val log881163 = sc.textFile("hdfs://ns1/user/songchunlin/881163.计算机应用.txt")
val rowRDDlog881163 = log881163.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
import sqlContext.implicits._
val slog881163 = rowRDDlog881163.map(line => (date, "881163", "计算机应用", line._4)).toDF("editdate", "number", "category", "words")
val s2881163 = slog881163.toDF("editdate", "number", "category", "words").registerTempTable("category")
val log881164 = sc.textFile("hdfs://ns1/user/songchunlin/881164.传媒.txt")
val rowRDDlog881164 = log881164.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881164 = rowRDDlog881164.map(line => (date, "881164", "传媒", line._4)).toDF("editdate", "number", "category", "words")
val log881165 = sc.textFile("hdfs://ns1/user/songchunlin/881165.综合.txt")
val rowRDDlog881165 = log881165.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881165 = rowRDDlog881165.map(line => (date, "881165", "综合", line._4)).toDF("editdate", "number", "category", "words")
val log881166 = sc.textFile("hdfs://ns1/user/songchunlin/881166.国防军工.txt")
val rowRDDlog881166 = log881166.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
val slog881166 = rowRDDlog881166.map(line => (date, "881166", "国防军工", line._4)).toDF("editdate", "number", "category", "words")
slog881101.insertIntoJDBC(rl,"category",false)
slog881102.insertIntoJDBC(rl,"category",false)
slog881103.insertIntoJDBC(rl,"category",false)
slog881104.insertIntoJDBC(rl,"category",false)
slog881105.insertIntoJDBC(rl,"category",false)
slog881106.insertIntoJDBC(rl,"category",false)
slog881107.insertIntoJDBC(rl,"category",false)
slog881108.insertIntoJDBC(rl,"category",false)
slog881109.insertIntoJDBC(rl,"category",false)
slog881110.insertIntoJDBC(rl,"category",false)
slog881111.insertIntoJDBC(rl,"category",false)
slog881112.insertIntoJDBC(rl,"category",false)
slog881113.insertIntoJDBC(rl,"category",false)
slog881114.insertIntoJDBC(rl,"category",false)
slog881115.insertIntoJDBC(rl,"category",false)
slog881116.insertIntoJDBC(rl,"category",false)
slog881117.insertIntoJDBC(rl,"category",false)
slog881118.insertIntoJDBC(rl,"category",false)
slog881119.insertIntoJDBC(rl,"category",false)
slog881120.insertIntoJDBC(rl,"category",false)
slog881121.insertIntoJDBC(rl,"category",false)
slog881122.insertIntoJDBC(rl,"category",false)
slog881123.insertIntoJDBC(rl,"category",false)
slog881124.insertIntoJDBC(rl,"category",false)
slog881125.insertIntoJDBC(rl,"category",false)
slog881126.insertIntoJDBC(rl,"category",false)
slog881127.insertIntoJDBC(rl,"category",false)
slog881128.insertIntoJDBC(rl,"category",false)
slog881129.insertIntoJDBC(rl,"category",false)
slog881130.insertIntoJDBC(rl,"category",false)
slog881131.insertIntoJDBC(rl,"category",false)
slog881132.insertIntoJDBC(rl,"category",false)
slog881133.insertIntoJDBC(rl,"category",false)
slog881134.insertIntoJDBC(rl,"category",false)
slog881135.insertIntoJDBC(rl,"category",false)
slog881136.insertIntoJDBC(rl,"category",false)
slog881137.insertIntoJDBC(rl,"category",false)
slog881138.insertIntoJDBC(rl,"category",false)
slog881139.insertIntoJDBC(rl,"category",false)
slog881140.insertIntoJDBC(rl,"category",false)
slog881141.insertIntoJDBC(rl,"category",false)
slog881142.insertIntoJDBC(rl,"category",false)
slog881143.insertIntoJDBC(rl,"category",false)
slog881144.insertIntoJDBC(rl,"category",false)
slog881145.insertIntoJDBC(rl,"category",false)
slog881146.insertIntoJDBC(rl,"category",false)
slog881147.insertIntoJDBC(rl,"category",false)
slog881148.insertIntoJDBC(rl,"category",false)
slog881149.insertIntoJDBC(rl,"category",false)
slog881150.insertIntoJDBC(rl,"category",false)
slog881151.insertIntoJDBC(rl,"category",false)
slog881152.insertIntoJDBC(rl,"category",false)
slog881153.insertIntoJDBC(rl,"category",false)
slog881154.insertIntoJDBC(rl,"category",false)
slog881155.insertIntoJDBC(rl,"category",false)
slog881156.insertIntoJDBC(rl,"category",false)
slog881157.insertIntoJDBC(rl,"category",false)
slog881158.insertIntoJDBC(rl,"category",false)
slog881159.insertIntoJDBC(rl,"category",false)
slog881160.insertIntoJDBC(rl,"category",false)
slog881161.insertIntoJDBC(rl,"category",false)
slog881162.insertIntoJDBC(rl,"category",false)
slog881163.insertIntoJDBC(rl,"category",false)
slog881164.insertIntoJDBC(rl,"category",false)
slog881165.insertIntoJDBC(rl,"category",false)
slog881166.insertIntoJDBC(rl,"category",false)
}
def insert(value0: String, value1: String, value2: String, value3: String): Unit = {
//val statement = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE)
// CREATE TABLE words2(innersessionId VARCHAR(100),words VARCHAR(100), VARCHAR(100),posit VARCHAR(100),va VARCHAR(100))
try {
val prep = conn.prepareStatement("INSERT INTO category(editdate,number,category,words) VALUES (?,?,?,?) ")
prep.setString(1, value0)
prep.setString(2, value1)
prep.setString(3, value2)
prep.setString(4, value3)
prep.executeUpdate
} catch {
case e: Exception => e.printStackTrace
}
}
def insert2(value0: String, value1: String, value2: String, value3: String): Unit = {
try {
val prep = conn.prepareStatement("INSERT INTO category2(editdate,number,category,words) VALUES (?,?,?,?) ")
prep.setString(1, value0)
prep.setString(2, value1)
prep.setString(3, value2)
prep.setString(4, value3)
prep.executeUpdate
} catch {
case e: Exception => e.printStackTrace
}
}
}
