Python往kafka生产消费数据

安装 kafka:  pip install kafka-python

生产数据

from kafka import KafkaProducer
import json
 
'''
    生产者demo
    向test_lyl2主题中循环写入10条json数据
    注意事项：要写入json数据需加上value_serializer参数，如下代码
'''
producer = KafkaProducer(
                            value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                            bootstrap_servers=['192.168.12.101:6667','192.168.12.102:6667','192.168.12.103:6667']
                         )
for i in range(10):
    data={
        "name":"李四",
        "age":23,
        "gender":"男",
        "id":i
    }
    producer.send('test_lyl2', data)
producer.close()

消费数据

from kafka import KafkaConsumer
import json
 
'''
    消费者demo
    消费test_lyl2主题中的数据
    注意事项：如需以json格式读取数据需加上value_deserializer参数
'''
 
 
consumer = KafkaConsumer('test_lyl2',group_id="lyl-gid1",
                         bootstrap_servers=['192.168.12.101:6667','192.168.12.102:6667','192.168.12.103:6667'],
                         auto_offset_reset='earliest',value_deserializer=json.loads
                         )
for message in consumer:
    print(message.value)