Storm定时和批处理

package com.storm.hbaseTest;

import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;

import backtype.storm.Config;
import backtype.storm.Constants;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.IRichBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Lists;

/** 38  * @version V1.0
 */
@SuppressWarnings("all")
public class HbaseBout implements IRichBolt{
    
      private static final long serialVersionUID = 1L;
      private static final Logger LOG = LoggerFactory.getLogger(HbaseBout.class);
      
      protected OutputCollector collector;
      protected HbaseClient hbaseClient;
      protected String tableName;
      protected String configKey ="hbase.conf";
      
      //批处理大小
      protected int batchSize = 15000;
      List<Put> batchMutations;
      List<Tuple> tupleBatch;
      //tick Time
      int flushIntervalSecs = 1;
      
    public void prepare(Map map, TopologyContext context,
            OutputCollector collector) {
        
        this.collector = collector;
        Configuration hbConfig = HBaseConfiguration.create();
        Map<String,String> conf = (Map)map.get(this.configKey);
        
        //获取Hbase配置
        if (conf == null) {
          throw new IllegalArgumentException("HBase configuration not found using key '" + this.configKey + "'");
        }
        
        //批大小
        if(map.get("batchSize") != null){
            this.batchSize = new Integer(map.get("batchSize").toString());
        }
        //Tick Time
        if(map.get("flushIntervalSecs") != null){
            this.flushIntervalSecs = Integer.valueOf(map.get("flushIntervalSecs").toString());
        }
        
        //Hbase 配置
        for (String key : conf.keySet()) {
          hbConfig.set(key, String.valueOf(conf.get(key)));
        }
        this.hbaseClient = new HbaseClient(hbConfig, this.tableName);
    }

    @Override
    public void execute(Tuple tuple) {
        
         boolean flush = false;
         
         try {
             //Tick
             if(tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID)&& tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID)){
                LOG.debug("TICK received! current batch status ", Integer.valueOf(this.tupleBatch.size()), Integer.valueOf(this.batchSize)); 
                flush = true;
            }else{
                
                Put put = new Put(tuple.getStringByField("rowKey").getBytes());
                put.add("cf".getBytes(), "name".getBytes(), tuple.getStringByField("name").getBytes());
                put.add("cf".getBytes(), "sex".getBytes(), tuple.getStringByField("sex").getBytes());
                this.batchMutations.add(put);
                this.tupleBatch.add(tuple);
                
                //当前tuple批大小
                if (this.tupleBatch.size() >= this.batchSize) {
                  flush = true;
                }
            }
             //持久化操作
             if ((flush) && (!this.tupleBatch.isEmpty())) {
                  this.hbaseClient.batchMutate(this.batchMutations);
                  LOG.debug("acknowledging tuples after batchMutate");
                  for (Iterator<Tuple> tuples = this.tupleBatch.iterator(); tuples.hasNext(); ) { 
                      Tuple t = (Tuple)tuples.next();
                        this.collector.ack(t);
                  }
                  this.tupleBatch.clear();
                  this.batchMutations.clear();
               }
        } catch (Exception e) {
              LOG.debug("inser batch fail");
              this.collector.reportError(e);
              for (Tuple t : this.tupleBatch) {
                  this.collector.fail(t);
                }
              this.tupleBatch.clear();
              this.batchMutations.clear();
        }
    }

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        
    }
    
    @Override
    public Map<String, Object> getComponentConfiguration() {
        Config conf = new Config();
        conf.put(conf.TOPOLOGY_TICK_TUPLE_FREQ_SECS,flushIntervalSecs);
        return conf;
    }

    
    @Override
    public void cleanup() {
        
    }
}

class HbaseClient{
     
     private static final Logger LOG = LoggerFactory.getLogger(HbaseClient.class);
      HConnection hTablePool = null;
      HTableInterface table = null;
      
      public HbaseClient(final Configuration configuration, final String tableName)
      {
        try
        {
            hTablePool =  HConnectionManager.createConnection(configuration) ;
            this.table =  hTablePool.getTable(tableName);
        }
        catch (Exception e) {
          throw new RuntimeException("HBase create failed: " + e.getMessage(), e);
        }
      }

      public void batchMutate(List<Put> puts) throws Exception {
          
          try {
            this.table.put(puts);
        } catch (Exception e) {
              LOG.warn("Error insert batch to HBase.", e);
              throw e;
        }
      }
}