kafka -client

1. 使用java连接kafka读取消息，首先将kafka安装包中libs中的所有jar导入到工程中，否则可能导致类库找不到的问题
 
 
package test.kafka;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;

public class kafkaCustomer {
    public static void main(String[] args) {
        String topic = "topoic";
        ConsumerConnector consumer = Consumer.createJavaConsumerConnector(createConsumerConfig());
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, new Integer(1));
        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
        KafkaStream<byte[], byte[]> stream =  consumerMap.get(topic).get(0);
        ConsumerIterator<byte[], byte[]> it = stream.iterator();
        while(it.hasNext())
            System.out.println("consume: " + new String(it.next().message()));
    }

    private static ConsumerConfig createConsumerConfig() {
        Properties props = new Properties();
        props.put("group.id","group1");
        props.put("zookeeper.connect","zookeeperaddip");
        props.put("zookeeper.session.timeout.ms", "4000");
        props.put("zookeeper.sync.time.ms", "2000");
        props.put("auto.commit.interval.ms", "1000");
        return new ConsumerConfig(props);

    }
}
 
2. python连接kafka 
  consumer - example
from kafka import KafkaConsumer
import json
import os
topic = ''
fw = open('log-test.txt','w')
consumer = KafkaConsumer(topic,group_id='a-my-group-1',bootstrap_servers=['127.0.0.1:9092'])
for message in consumer:
    try:
        data =  message[6]
        print data
        d = json.loads(data)
        fw.write(json.dumps(d) + '\n')
    except Exception,e:
        print str(e)
        print(message)
fw.close()
 
produce - example
 
from kafka import KafkaProducer
import json
import time
 
topic = ''
producer = KafkaProducer(bootstrap_servers=['10.1.1.1:9092'])
for i in range(100):
    d = {}
    d['rm'] = '/2/ok-1' + str(i)
    print json.dumps(d)
    producer.send(topic, json.dumps(d))
    producer.flush()
    time.sleep(0.1)