kafka学习笔记之生产者与消费者的实现

 

 第一次写博客啊！谢谢大家支持，转载请注明地址http://www.cnblogs.com/zzl198949/articles/3823264.html

最近，由于工作原因，需要使用kafka，花了几天时间大概了解kafka安装以及生产者（producer）、消费者（consumer）的工作原理。同时也实现了生产者生成消息和消费者接收消息的代码。

但是发现的问题是消费者接收到的byte[]数组转化为String时没有问题，但是转换为对象时发现总是报错，后来发现应该是kafka内部序列化的时候获取byte[]方法没有用ObjectInputStream导致的，所以使用ObjectOutputStream当然不能把byte[]转为Object，下面先贴一下我自定义的对象People以及Object与byte[]互转的代码

 1 package com.fh.netpf.kafka.consumer;
 2 
 3 import java.io.Serializable;
 4 
 5 import org.apache.commons.lang.builder.ReflectionToStringBuilder;
 6 import org.apache.commons.lang.builder.ToStringStyle;
 7 
 8 public class People implements Serializable {
 9     /**
10      * 
11      */
12     private static final long serialVersionUID = -2942498587223679355L;
13 
14     public String name;
15 
16     public String sex;
17 
18     public String getName() {
19         return name;
20     }
21 
22     public void setName(String name) {
23         this.name = name;
24     }
25 
26     public String getSex() {
27         return sex;
28     }
29 
30     public void setSex(String sex) {
31         this.sex = sex;
32     }
33 
34     /** {@inheritDoc} */
35 
36     public String toString() {
37         return ReflectionToStringBuilder.toString(this,
38                 ToStringStyle.SHORT_PREFIX_STYLE);
39     }
40 }
 1 package com.fh.netpf.kafka.consumer;
 2 
 3 import java.io.ByteArrayInputStream;
 4 import java.io.ByteArrayOutputStream;
 5 import java.io.IOException;
 6 import java.io.ObjectInputStream;
 7 import java.io.ObjectOutputStream;
 8 import java.io.Serializable;
 9 
10 import org.apache.commons.lang.builder.ReflectionToStringBuilder;
11 import org.apache.commons.lang.builder.ToStringStyle;
12 
13 import com.fh.netpf.msb.structs.Message;
14 
15 public class Object2BytesUtils {
16 
17     public static People ByteToObject(byte[] item) {
18         People people = null;
19         ByteArrayInputStream byteIn = null;
20         ObjectInputStream oi = null;
21         try {
22             byteIn = new ByteArrayInputStream(item);
23             oi = new ObjectInputStream(byteIn);
24             people = (People) oi.readObject();
25         } catch (Exception e) {
26             e.printStackTrace();
27         } finally {
28             try {
29                 byteIn.close();
30                 oi.close();
31             } catch (IOException e) {
32                 e.printStackTrace();
33             }
34         }
35         return people;
36     }
37 
38     public static byte[] ObjectToByte(People obj) {
39         byte[] bytes = null;
40         ByteArrayOutputStream bo = null;
41         ObjectOutputStream oo = null;
42         try {
43             bo = new ByteArrayOutputStream();
44             oo = new ObjectOutputStream(bo);
45             oo.writeObject(obj);
46             bytes = bo.toByteArray();
47         } catch (Exception e) {
48             e.printStackTrace();
49         } finally {
50             try {
51                 bo.close();
52                 oo.close();
53             } catch (IOException e) {
54             }
55         }
56         return bytes;
57     }
58 
59     public static void main(String[] args) {
60         People people = new People();
61         people.setName("zzl");
62         people.setSex("男");
63         System.out.print(ByteToObject(ObjectToByte(people)));
64     }
65 
66 }

由于kafka是使用scala写的，本着java还没学好学什么scala的心理就没想着去找kafka自己获取byte[]的方法，经过一天的折腾终于找到了办法，就是实现kafka.serializer.Encoder接口用自己的获取byte[]的方法代替，话不多说，先上代码MyEncoder

 1 package com.fh.netpf.kafka.message;
 2 
 3 import kafka.utils.VerifiableProperties;
 4 
 5 import com.fh.netpf.kafka.consumer.Object2BytesUtils;
 6 import com.fh.netpf.kafka.consumer.People;
 7 
 8 public class MyEncoder implements kafka.serializer.Encoder<People> {
 9 
10     public String encoding;
11 
12     public MyEncoder(VerifiableProperties props) {
13         if (props == null) {
14             encoding = "UTF8";
15         } else {
16             props.getString("serializer.encoding", "UTF8");
17         }
18     }
19 
20     public byte[] toBytes(People people) {
21         return Object2BytesUtils.ObjectToByte(people);
22     }
23 }

剩下需要做的就是在producer生产消息时使用com.fh.netpf.kafka.message.MyEncoder作为获取byte[]的方法，就是在ProducerConfig中设置serializer.class为com.fh.netpf.kafka.message.MyEncoder，最后消费者获取byte[]直接调用ByteToObject获取People对象，下面是生成者和消费者的代码
producer

 1 package com.fh.netpf.kafka.producer;
 2 
 3 import java.util.ArrayList;
 4 import java.util.Collection;
 5 import java.util.List;
 6 import java.util.Properties;
 7 
 8 import kafka.javaapi.producer.Producer;
 9 import kafka.producer.KeyedMessage;
10 import kafka.producer.ProducerConfig;
11 
12 import com.fh.netpf.kafka.consumer.People;
13 
14 public class ProducerSample {
15 
16     private Producer<String, People> inner;
17 
18     public ProducerSample() throws Exception {
19         Properties properties = new Properties();
20         properties.load(ClassLoader
21                 .getSystemResourceAsStream("producer.properties"));
22         ProducerConfig config = new ProducerConfig(properties);
23         inner = new Producer<String, People>(config);
24     }
25 
26     public void send(String topicName, People people) {
27         if (topicName == null || people == null) {
28             return;
29         }
30         KeyedMessage<String, People> km = new KeyedMessage<String, People>(
31                 topicName, people);
32         inner.send(km);
33     }
34 
35     public void send(String topicName, Collection<People> peoples) {
36         if (topicName == null || peoples == null) {
37             return;
38         }
39         if (peoples.isEmpty()) {
40             return;
41         }
42         List<KeyedMessage<String, People>> kms = new ArrayList<KeyedMessage<String, People>>();
43         for (People entry : peoples) {
44             KeyedMessage<String, People> km = new KeyedMessage<String, People>(
45                     topicName, entry);
46             kms.add(km);
47         }
48         inner.send(kms);
49     }
50 
51     public void close() {
52         inner.close();
53     }
54 
55     /**
56      * @param args
57      */
58     public static void main(String[] args) {
59         ProducerSample producer = null;
60         try {
61             producer = new ProducerSample();
62             People people = new People();
63             people.setName("zlzhao");
64             people.setSex("男");
65             producer.send("test", people);
66         } catch (Exception e) {
67             e.printStackTrace();
68         } finally {
69             if (producer != null) {
70                 producer.close();
71             }
72         }
73     }
74 }

producer.properties

zookeeper.connect=192.168.1.10:2181
##serializer.class=kafka.serializer.StringEncoder
serializer.class=com.fh.netpf.kafka.message.MyEncoder
metadata.broker.list=192.168.1.10:9092 producer.type=sync compression.codec=0

consumer:

  1 package com.fh.netpf.kafka.consumer;
  2 
  3 import java.util.HashMap;
  4 import java.util.List;
  5 import java.util.Map;
  6 import java.util.Properties;
  7 import java.util.concurrent.ExecutorService;
  8 import java.util.concurrent.Executors;
  9 
 10 import kafka.consumer.Consumer;
 11 import kafka.consumer.ConsumerConfig;
 12 import kafka.consumer.ConsumerIterator;
 13 import kafka.consumer.KafkaStream;
 14 import kafka.javaapi.consumer.ConsumerConnector;
 15 import kafka.message.MessageAndMetadata;
 16 
 17 public class ConsumerSample {
 18 
 19     private ConsumerConfig config;
 20     private String topic;
 21     private int partitionsNum;
 22     private MessageExecutor executor;
 23     private ConsumerConnector connector;
 24     private ExecutorService threadPool;
 25 
 26     public ConsumerSample(String topic, int partitionsNum,
 27             MessageExecutor executor) throws Exception {
 28         Properties properties = new Properties();
 29         properties.load(ClassLoader
 30                 .getSystemResourceAsStream("consumer.properties"));
 31         config = new ConsumerConfig(properties);
 32         this.topic = topic;
 33         this.partitionsNum = partitionsNum;
 34         this.executor = executor;
 35     }
 36 
 37     public void start() throws Exception {
 38         connector = Consumer.createJavaConsumerConnector(config);
 39         Map<String, Integer> topics = new HashMap<String, Integer>();
 40         topics.put(topic, partitionsNum);
 41         Map<String, List<KafkaStream<byte[], byte[]>>> streams = connector
 42                 .createMessageStreams(topics);
 43         List<KafkaStream<byte[], byte[]>> partitions = streams.get(topic);
 44         threadPool = Executors.newFixedThreadPool(partitionsNum);
 45         for (KafkaStream<byte[], byte[]> partition : partitions) {
 46             threadPool.execute(new MessageRunner(partition));
 47         }
 48     }
 49 
 50     public void close() {
 51         try {
 52             threadPool.shutdownNow();
 53         } catch (Exception e) {
 54             //
 55         } finally {
 56             connector.shutdown();
 57         }
 58 
 59     }
 60 
 61     class MessageRunner implements Runnable {
 62         private KafkaStream<byte[], byte[]> partition;
 63 
 64         MessageRunner(KafkaStream<byte[], byte[]> partition) {
 65             this.partition = partition;
 66         }
 67 
 68         public void run() {
 69             ConsumerIterator<byte[], byte[]> it = partition.iterator();
 70             while (it.hasNext()) {
 71                 MessageAndMetadata<byte[], byte[]> item = it.next();
 72                 System.out.println("partiton:" + item.partition());
 73                 System.out.println("offset:" + item.offset());
 74                 executor.execute(Object2BytesUtils.ByteToObject(item.message()));// UTF-8
 75 
 76             }
 77         }
 78     }
 79 
 80     interface MessageExecutor {
 81 
 82         public void execute(People people);
 83     }
 84 
 85     /**
 86      * @param args
 87      */
 88     public static void main(String[] args) {
 89         ConsumerSample consumer = null;
 90         try {
 91             MessageExecutor executor = new MessageExecutor() {
 92 
 93                 public void execute(People people) {
 94                     System.out.println(people);
 95                 }
 96             };
 97             consumer = new ConsumerSample("test",
 98                     2, executor);
 99             consumer.start();
100         } catch (Exception e) {
101             e.printStackTrace();
102         } finally {
103             // if(consumer != null){
104             // consumer.close();
105             // }
106         }
107     }
108 
109 }

consumer.properties

zookeeper.connect=192.168.1.10:2181
##,127.0.0.1:2182,127.0.0.1:2183
# timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=150000
#consumer group id
group.id=test
#consumer timeout
#consumer.timeout.ms=5000

以上就是整个的实现过程，欢迎大家提意见，当然也希望大家能找到更好解决办法







posted on 2014-07-03 20:04  smile198949  阅读(370)  评论(0)    收藏  举报