package com.asp;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
public class HFileGenerator {
public static Configuration conf = HBaseConfiguration.create();
static {
Configuration HBASE_CONFIG = new Configuration();
HBASE_CONFIG.set("hbase.zookeeper.quorum",
"BJ-YZ-103R-63-38,BJ-YZ-103R-63-39,BJ-YZ-103R-63-40");
HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", "2181");
conf = HBaseConfiguration.create(HBASE_CONFIG);
}
public static class HourlyImporter extends
Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] items = line.split("@_@");
if (items.length < 52) {
return;
}
String[] cols = { "name", "BRAND", "PROVINCE", "CITY", "ADDR",
"SEX", "AGE", "GROUP_NAME", "CHILD", "CAR", "CAR_ID",
"SVIP", "FINANACE", "BOSS", "ABROAD", "HIGH_VALUE", "VIP",
"UA_BRAND", "UA_TYPE", "UA_OS", "UA_DATE", "BROWSER",
"SMS_USER", "MMS_USER", "WAP_USER", "APP_USER",
"CURR_ARPU", "TOTAL_ARPU", "WAP_LAST_DATE",
"WAP_LAST_SUB_DATE", "WAP_CURR_DAYS", "WAP_TOTAL_ARPU",
"CXP_DURATION", "CXP_IS_OFF", "CXP_IS_ACTIVE",
"CXP_CURR_ARPU", "TV_SG_NUM", "TV_KEY_NUM", "TV_SUB_DATE",
"TV_SUB_CHANNEL", "TV_SERVICE_ID", "TY_EXP_NUM",
"TY_ORDER_NUM", "TY_TOTAL_EXP_NUM", "TY_TOTAL_ORDER_NUM",
"SQ_USER", "SQ_RECV_NUM", "SQ_AREA", "WAP_FEATURE_STR",
"CXP_FEATURE_STR", "KEYWORD_STR" };
byte[] bRowKey = Bytes.toBytes(items[0]);
ImmutableBytesWritable rowkey = new ImmutableBytesWritable(bRowKey);
Put put = new Put(bRowKey);
for (int i = 0; i < cols.length; i++) {
if (items[i+1].equals(null) || items[i+1].equals("")) {
continue;
}
put.add("c1".getBytes(), cols[i].getBytes(), items[i+1].getBytes());
}
context.write(rowkey, put);
}
}
public static Job createSubmittableJob(Configuration conf, String[] args)
throws IOException {
String tableName = args[0];
Path inputDir = new Path(args[1]);
Job job = new Job(conf, "hello");
job.setJarByClass(HFileGenerator.class);
FileInputFormat.setInputPaths(job, inputDir);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(HourlyImporter.class);
TableMapReduceUtil.initTableReducerJob(tableName, null, job);
job.setNumReduceTasks(0);
TableMapReduceUtil.addDependencyJars(job);
return job;
}
public static void main(String[] args) throws Exception {
Job job = createSubmittableJob(conf, args);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}